Support dynamic scales for Qconv's and Dequantize (#2171)

* Support dynamic scales for Qconv's and Dequantize

* Remove constant folding

* add additional dynamic_quantize unittest

* add another mxnet quantize unittest

* add additional dynamic_dequantize tests

* fix shape error

* add dynamic signed_quantize unittest

* Pass correct scale

* Refactoring

* Added dynamic scale support for QCBA and QCBSA

* Refactor to create MKLDNN primitives on the first iteration

* remove stray code

* unused variables

* remove extraneous line

src/ngraph/op/experimental/quantized_conv_relu.hpp

@@ -45,6 +45,7 @@ namespace ngraph
             const Strides& get_data_dilation_strides() const { return m_data_dilation_strides; }
             std::shared_ptr<Node> get_filters() { return get_argument(1); }
             std::shared_ptr<Node> get_data_batch() { return get_argument(0); }
+            bool with_relu() const { return true; }
             virtual std::shared_ptr<Node>
                 copy_with_new_args(const NodeVector& new_args) const override;
 

src/ngraph/runtime/cpu/builder/quantization.cpp

@@ -52,16 +52,57 @@ namespace ngraph
                     auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
                     auto input_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
                     auto result_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
-                    size_t dequantize_index =
-                        mkldnn_emitter->build_dequantization(node, input_desc, result_desc);
-                    auto& deps = mkldnn_emitter->get_primitive_deps(dequantize_index);
-                    functor = [&, dequantize_index](CPURuntimeContext* ctx,
-                                                    CPUExecutionContext* ectx) {
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], out_tensor);
-                        cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, dequantize_index);
-                    };
-                    functors.emplace_back(functor);
+                    auto scale_const_op = std::dynamic_pointer_cast<ngraph::op::Constant>(
+                        dequantize->get_argument(1));
+                    std::vector<float> scales;
+                    if (scale_const_op == nullptr)
+                    {
+                        auto& arg1_tensor = external_function->get_tensor_data(args[1].get_name());
+                        auto scales_size = shape_size(args[1].get_shape());
+
+                        size_t dequantize_index =
+                            mkldnn_emitter->build_dequantization(node, input_desc, result_desc);
+                        auto& deps = mkldnn_emitter->get_primitive_deps(dequantize_index);
+                        functor = [&, input_desc, result_desc, scales_size, dequantize_index](
+                            CPURuntimeContext* ctx, CPUExecutionContext* ectx) {
+                            // Create MKLDNN reorder primitive during the first iteration.
+                            // Assumes the scales dont change for the duration of the graph
+                            if (ctx->first_iteration)
+                            {
+                                mkldnn::primitive_attr attr;
+                                vector<float> dyn_scales;
+                                dyn_scales.assign(static_cast<float*>(arg1_tensor),
+                                                  static_cast<float*>(arg1_tensor) + scales_size);
+                                attr.set_output_scales(0, dyn_scales);
+                                attr.set_int_output_round_mode(mkldnn::round_mode::round_nearest);
+                                auto reorder_desc = mkldnn::reorder::primitive_desc(
+                                    {input_desc, executor::global_cpu_engine},
+                                    {result_desc, executor::global_cpu_engine},
+                                    attr);
+                                *ctx->mkldnn_primitives[dequantize_index] =
+                                    mkldnn::reorder(reorder_desc,
+                                                    *ctx->mkldnn_primitives[deps[0]],
+                                                    *ctx->mkldnn_primitives[deps[1]]);
+                            }
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], out_tensor);
+                            cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, dequantize_index);
+                        };
+                        functors.emplace_back(functor);
+                    }
+                    else
+                    {
+                        size_t dequantize_index =
+                            mkldnn_emitter->build_dequantization(node, input_desc, result_desc);
+                        auto& deps = mkldnn_emitter->get_primitive_deps(dequantize_index);
+                        functor = [&, dequantize_index](CPURuntimeContext* ctx,
+                                                        CPUExecutionContext* ectx) {
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], out_tensor);
+                            cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, dequantize_index);
+                        };
+                        functors.emplace_back(functor);
+                    }
                 }
                 else
                 {
@@ -223,6 +264,7 @@ namespace ngraph
                                 vector<float> dyn_scales;
                                 dyn_scales.assign(static_cast<float*>(arg1_tensor),
                                                   static_cast<float*>(arg1_tensor) + scales_size);
+                                dyn_scales[0] = 1.0 / dyn_scales[0];
                                 attr.set_output_scales(0, dyn_scales);
                                 attr.set_int_output_round_mode(mkldnn::round_mode::round_nearest);
                                 auto reorder_desc = mkldnn::reorder::primitive_desc(

src/ngraph/runtime/cpu/builder/quantized_conv.cpp

@@ -19,6 +19,7 @@
 #include "ngraph/op/experimental/quantized_conv_bias.hpp"
 #include "ngraph/op/experimental/quantized_conv_relu.hpp"
 #include "ngraph/runtime/cpu/cpu_builder.hpp"
+#include "ngraph/runtime/cpu/cpu_executor.hpp"
 #include "ngraph/runtime/cpu/mkldnn_invoke.hpp"
 #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
 
@@ -39,17 +40,35 @@ namespace ngraph
                     auto& functors = external_function->get_functors();
                     auto& arg0_tensor = external_function->get_tensor_data(args[0].get_name());
                     auto& arg1_tensor = external_function->get_tensor_data(args[1].get_name());
+                    auto& arg2_tensor = external_function->get_tensor_data(args[2].get_name());
                     auto& out0_tensor = external_function->get_tensor_data(out[0].get_name());
 
                     auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
+                    auto scales_size = shape_size(args[2].get_shape());
 
-                    auto conv_index =
-                        mkldnn_emitter->build_convolution<ngraph::op::QuantizedConvolution>(
-                            node, args, out);
+                    auto conv_desc =
+                        mkldnn_emitter
+                            ->get_convolution_forward_desc<ngraph::op::QuantizedConvolution>(
+                                node, args, out);
+                    auto conv_attr =
+                        mkldnn_emitter
+                            ->get_convolution_forward_attr<ngraph::op::QuantizedConvolution>(node);
+                    size_t conv_index = mkldnn_emitter->convolution_forward_init();
                     auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
 
-                    auto functor = [&, conv_index](CPURuntimeContext* ctx,
-                                                   CPUExecutionContext* ectx) {
+                    auto functor = [&, scales_size, conv_desc, conv_attr, deps, conv_index](
+                        CPURuntimeContext* ctx, CPUExecutionContext* ectx) mutable {
+                        // Create MKLDNN convolution primitive during the first iteration.
+                        // Assumes the scales dont change for the duration of the graph
+                        if (ctx->first_iteration)
+                        {
+                            vector<float> dyn_scales;
+                            dyn_scales.assign(static_cast<float*>(arg2_tensor),
+                                              static_cast<float*>(arg2_tensor) + scales_size);
+                            conv_attr.set_output_scales(0, dyn_scales);
+                            mkldnn_emitter->convolution_forward<false>(
+                                conv_desc, conv_attr, executor::global_cpu_engine, conv_index);
+                        }
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], out0_tensor);
@@ -71,17 +90,34 @@ namespace ngraph
                     auto& functors = external_function->get_functors();
                     auto& arg0_tensor = external_function->get_tensor_data(args[0].get_name());
                     auto& arg1_tensor = external_function->get_tensor_data(args[1].get_name());
+                    auto& arg2_tensor = external_function->get_tensor_data(args[2].get_name());
                     auto& out0_tensor = external_function->get_tensor_data(out[0].get_name());
 
                     auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
+                    auto scales_size = shape_size(args[2].get_shape());
 
-                    auto conv_index =
-                        mkldnn_emitter->build_convolution<ngraph::op::QuantizedConvolutionRelu>(
-                            node, args, out);
+                    auto conv_desc =
+                        mkldnn_emitter
+                            ->get_convolution_forward_desc<ngraph::op::QuantizedConvolutionRelu>(
+                                node, args, out);
+                    auto conv_attr =
+                        mkldnn_emitter
+                            ->get_convolution_forward_attr<ngraph::op::QuantizedConvolutionRelu>(
+                                node);
+                    size_t conv_index = mkldnn_emitter->convolution_forward_init();
                     auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
 
-                    auto functor = [&, conv_index](CPURuntimeContext* ctx,
-                                                   CPUExecutionContext* ectx) {
+                    auto functor = [&, scales_size, conv_desc, conv_attr, deps, conv_index](
+                        CPURuntimeContext* ctx, CPUExecutionContext* ectx) mutable {
+                        if (ctx->first_iteration)
+                        {
+                            vector<float> dyn_scales;
+                            dyn_scales.assign(static_cast<float*>(arg2_tensor),
+                                              static_cast<float*>(arg2_tensor) + scales_size);
+                            conv_attr.set_output_scales(0, dyn_scales);
+                            mkldnn_emitter->convolution_forward<false>(
+                                conv_desc, conv_attr, executor::global_cpu_engine, conv_index);
+                        }
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], out0_tensor);
@@ -105,16 +141,34 @@ namespace ngraph
                     auto& arg0_tensor = external_function->get_tensor_data(args[0].get_name());
                     auto& arg1_tensor = external_function->get_tensor_data(args[1].get_name());
                     auto& arg2_tensor = external_function->get_tensor_data(args[2].get_name());
+                    auto& arg3_tensor = external_function->get_tensor_data(args[3].get_name());
                     auto& out0_tensor = external_function->get_tensor_data(out[0].get_name());
 
                     auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
-                    auto conv_index =
-                        mkldnn_emitter->build_convolution<ngraph::op::QuantizedConvolutionBias>(
-                            node, args, out);
+                    auto scales_size = shape_size(args[3].get_shape());
+
+                    auto conv_desc =
+                        mkldnn_emitter
+                            ->get_convolution_forward_desc<ngraph::op::QuantizedConvolutionBias>(
+                                node, args, out);
+                    auto conv_attr =
+                        mkldnn_emitter
+                            ->get_convolution_forward_attr<ngraph::op::QuantizedConvolutionBias>(
+                                node);
+                    size_t conv_index = mkldnn_emitter->convolution_forward_init(true);
                     auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
 
-                    auto functor = [&, conv_index](CPURuntimeContext* ctx,
-                                                   CPUExecutionContext* ectx) {
+                    auto functor = [&, scales_size, conv_desc, conv_attr, deps, conv_index](
+                        CPURuntimeContext* ctx, CPUExecutionContext* ectx) mutable {
+                        if (ctx->first_iteration)
+                        {
+                            vector<float> dyn_scales;
+                            dyn_scales.assign(static_cast<float*>(arg3_tensor),
+                                              static_cast<float*>(arg3_tensor) + scales_size);
+                            conv_attr.set_output_scales(0, dyn_scales);
+                            mkldnn_emitter->convolution_forward<true>(
+                                conv_desc, conv_attr, executor::global_cpu_engine, conv_index);
+                        }
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
                         cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], arg2_tensor);
@@ -139,22 +193,64 @@ namespace ngraph
                     auto& arg0_tensor = external_function->get_tensor_data(args[0].get_name());
                     auto& arg1_tensor = external_function->get_tensor_data(args[1].get_name());
                     auto& arg2_tensor = external_function->get_tensor_data(args[2].get_name());
+                    auto& arg4_tensor = external_function->get_tensor_data(args[4].get_name());
+                    auto& arg5_tensor = external_function->get_tensor_data(args[5].get_name());
                     auto& out0_tensor = external_function->get_tensor_data(out[0].get_name());
 
                     auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
-                    auto conv_index =
-                        mkldnn_emitter->build_convolution<ngraph::op::QuantizedConvolutionBiasAdd>(
-                            node, args, out);
+                    auto scales_size = shape_size(args[4].get_shape());
+                    auto sum_scales_size = shape_size(args[5].get_shape());
+
+                    auto conv_desc =
+                        mkldnn_emitter
+                            ->get_convolution_forward_desc<ngraph::op::QuantizedConvolutionBiasAdd>(
+                                node, args, out);
+                    auto conv_attr =
+                        mkldnn_emitter
+                            ->get_convolution_forward_attr<ngraph::op::QuantizedConvolutionBiasAdd>(
+                                node);
+                    size_t conv_index = mkldnn_emitter->convolution_forward_init(true);
                     auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
 
-                    auto functor = [&, conv_index](CPURuntimeContext* ctx,
-                                                   CPUExecutionContext* ectx) {
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], arg2_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[3], out0_tensor);
-                        cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, conv_index);
-                    };
+                    auto functor =
+                        [&, scales_size, sum_scales_size, conv_desc, conv_attr, deps, conv_index](
+                            CPURuntimeContext* ctx, CPUExecutionContext* ectx) mutable {
+                            if (ctx->first_iteration)
+                            {
+                                vector<float> dyn_scales;
+                                vector<float> dyn_post_op_scales;
+                                dyn_scales.assign(static_cast<float*>(arg4_tensor),
+                                                  static_cast<float*>(arg4_tensor) + scales_size);
+                                dyn_post_op_scales.assign(static_cast<float*>(arg5_tensor),
+                                                          static_cast<float*>(arg5_tensor) +
+                                                              sum_scales_size);
+                                auto old_pops = conv_attr.get_post_ops();
+                                mkldnn::post_ops new_pops;
+                                for (int i = 0; i < old_pops.len(); i++)
+                                {
+                                    if (old_pops.kind(i) == mkldnn::primitive::kind::eltwise)
+                                    {
+                                        mkldnn::algorithm alg;
+                                        float scale, alpha, beta;
+                                        old_pops.get_params_eltwise(i, scale, alg, alpha, beta);
+                                        new_pops.append_eltwise(scale, alg, alpha, beta);
+                                    }
+                                    if (old_pops.kind(i) == mkldnn::primitive::kind::sum)
+                                    {
+                                        new_pops.append_sum(dyn_post_op_scales[0]);
+                                    }
+                                }
+                                conv_attr.set_output_scales(0, dyn_scales);
+                                conv_attr.set_post_ops(new_pops);
+                                mkldnn_emitter->convolution_forward<true>(
+                                    conv_desc, conv_attr, executor::global_cpu_engine, conv_index);
+                            }
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], arg2_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[3], out0_tensor);
+                            cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, conv_index);
+                        };
                     functors.emplace_back(functor);
                 }
                 else
@@ -173,23 +269,60 @@ namespace ngraph
                     auto& arg0_tensor = external_function->get_tensor_data(args[0].get_name());
                     auto& arg1_tensor = external_function->get_tensor_data(args[1].get_name());
                     auto& arg2_tensor = external_function->get_tensor_data(args[2].get_name());
+                    auto& arg4_tensor = external_function->get_tensor_data(args[4].get_name());
+                    auto& arg5_tensor = external_function->get_tensor_data(args[5].get_name());
                     auto& out0_tensor = external_function->get_tensor_data(out[0].get_name());
 
                     auto& mkldnn_emitter = external_function->get_mkldnn_emitter();
-                    auto conv_index =
-                        mkldnn_emitter
-                            ->build_convolution<ngraph::op::QuantizedConvolutionBiasSignedAdd>(
-                                node, args, out);
+                    auto scales_size = shape_size(args[4].get_shape());
+                    auto sum_scales_size = shape_size(args[5].get_shape());
+
+                    auto conv_desc = mkldnn_emitter->get_convolution_forward_desc<
+                        ngraph::op::QuantizedConvolutionBiasSignedAdd>(node, args, out);
+                    auto conv_attr = mkldnn_emitter->get_convolution_forward_attr<
+                        ngraph::op::QuantizedConvolutionBiasSignedAdd>(node);
+                    size_t conv_index = mkldnn_emitter->convolution_forward_init(true);
                     auto& deps = mkldnn_emitter->get_primitive_deps(conv_index);
 
-                    auto functor = [&, conv_index](CPURuntimeContext* ctx,
-                                                   CPUExecutionContext* ectx) {
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], arg2_tensor);
-                        cpu::mkldnn_utils::set_memory_ptr(ctx, deps[3], out0_tensor);
-                        cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, conv_index);
-                    };
+                    auto functor =
+                        [&, scales_size, sum_scales_size, conv_desc, conv_attr, deps, conv_index](
+                            CPURuntimeContext* ctx, CPUExecutionContext* ectx) mutable {
+                            if (ctx->first_iteration)
+                            {
+                                vector<float> dyn_scales;
+                                vector<float> dyn_post_op_scales;
+                                dyn_scales.assign(static_cast<float*>(arg4_tensor),
+                                                  static_cast<float*>(arg4_tensor) + scales_size);
+                                dyn_post_op_scales.assign(static_cast<float*>(arg5_tensor),
+                                                          static_cast<float*>(arg5_tensor) +
+                                                              sum_scales_size);
+                                auto old_pops = conv_attr.get_post_ops();
+                                mkldnn::post_ops new_pops;
+                                for (int i = 0; i < old_pops.len(); i++)
+                                {
+                                    if (old_pops.kind(i) == mkldnn::primitive::kind::eltwise)
+                                    {
+                                        mkldnn::algorithm alg;
+                                        float scale, alpha, beta;
+                                        old_pops.get_params_eltwise(i, scale, alg, alpha, beta);
+                                        new_pops.append_eltwise(scale, alg, alpha, beta);
+                                    }
+                                    if (old_pops.kind(i) == mkldnn::primitive::kind::sum)
+                                    {
+                                        new_pops.append_sum(2 * dyn_post_op_scales[0]);
+                                    }
+                                }
+                                conv_attr.set_post_ops(new_pops);
+                                conv_attr.set_output_scales(0, dyn_scales);
+                                mkldnn_emitter->convolution_forward<true>(
+                                    conv_desc, conv_attr, executor::global_cpu_engine, conv_index);
+                            }
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[0], arg0_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[1], arg1_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[2], arg2_tensor);
+                            cpu::mkldnn_utils::set_memory_ptr(ctx, deps[3], out0_tensor);
+                            cpu::mkldnn_utils::mkldnn_invoke_primitive(ctx, conv_index);
+                        };
                     functors.emplace_back(functor);
                 }
                 else

src/ngraph/runtime/cpu/mkldnn_emitter.cpp

@@ -148,10 +148,14 @@ size_t MKLDNNEmitter::build_dequantization(const ngraph::Node* node,
 {
     auto dequantize = static_cast<const ngraph::op::Dequantize*>(node);
     auto scale_const_op =
-        std::static_pointer_cast<ngraph::op::Constant>(dequantize->get_argument(1));
-    float scale = *(static_cast<float const*>(scale_const_op->get_data_ptr()));
+        std::dynamic_pointer_cast<ngraph::op::Constant>(dequantize->get_argument(1));
+    std::vector<float> scale = {1.0f};
+    if (scale_const_op != nullptr)
+    {
+        scale = scale_const_op->get_vector<float>();
+    }
     std::vector<float> scales;
-    scales.push_back(scale);
+    scales.push_back(scale[0]);
     size_t dequantize_index = 0;
     dequantize_index = this->build_quantize_reorder(input_desc, result_desc, scales);
     return dequantize_index;
@@ -1203,3 +1207,21 @@ size_t MKLDNNEmitter::build_bounded_relu(const mkldnn::memory::desc& input_desc,
     m_primitive_deps[primitive_index] = {input_index, result_index};
     return primitive_index;
 }
+
+size_t MKLDNNEmitter::convolution_forward_init(bool with_bias)
+{
+    size_t size = m_mkldnn_primitives.size();
+    if (with_bias)
+    {
+        // Inputs, Weights, Bias, Results, Conv
+        m_mkldnn_primitives.resize(size + 5, nullptr);
+        m_primitive_deps[m_mkldnn_primitives.size() - 1] = {size, size + 1, size + 2, size + 3};
+    }
+    else
+    {
+        // Inputs, Weights, Results, Conv
+        m_mkldnn_primitives.resize(size + 4, nullptr);
+        m_primitive_deps[m_mkldnn_primitives.size() - 1] = {size, size + 1, size + 2};
+    }
+    return m_mkldnn_primitives.size() - 1;
+}

src/ngraph/runtime/cpu/mkldnn_emitter.hpp

@@ -30,6 +30,7 @@
 #include "ngraph/op/experimental/quantized_conv.hpp"
 #include "ngraph/op/experimental/quantized_conv_bias.hpp"
 #include "ngraph/op/experimental/quantized_conv_relu.hpp"
+#include "ngraph/runtime/cpu/cpu_executor.hpp"
 #include "ngraph/runtime/cpu/cpu_tensor_view_wrapper.hpp"
 #include "ngraph/runtime/cpu/mkldnn_utils.hpp"
 #include "ngraph/runtime/cpu/op/bounded_relu.hpp"
@@ -40,6 +41,8 @@
 #include "ngraph/strides.hpp"
 #include "ngraph/type/element_type.hpp"
 
+#define MKLDNN_DIMS(X) mkldnn::memory::dims(X.begin(), X.end())
+
 namespace ngraph
 {
     namespace runtime
@@ -135,15 +138,15 @@ namespace ngraph
                 template <typename OP>
                 std::vector<float> extract_scale_value(const ngraph::Node* node, int index)
                 {
-                    auto qc = dynamic_cast<const OP*>(node);
+                    auto qc = static_cast<const OP*>(node);
+                    std::vector<float> scale_val = {1.0f};
                     auto scale_const_op =
                         std::dynamic_pointer_cast<ngraph::op::Constant>(qc->get_arguments()[index]);
-                    if (scale_const_op == nullptr)
+                    if (scale_const_op != nullptr)
                     {
-                        throw ngraph_error("Scale must be a Constant");
+                        scale_val = scale_const_op->template get_vector<float>();
                     }
 
-                    auto scale_val = scale_const_op->template get_vector<float>();
                     return scale_val;
                 }
 
@@ -197,53 +200,7 @@ namespace ngraph
                         ops.append_sum(2.0 * sum_scale_val[0]);
                     }
 
-                    auto add_relu = [&]() {
-                        if (dynamic_cast<const ngraph::op::ConvolutionBias*>(node))
-                        {
-                            return (dynamic_cast<const ngraph::op::ConvolutionBias*>(node))
-                                ->with_relu();
-                        }
-                        if (dynamic_cast<const ngraph::op::ConvolutionBiasAdd*>(node))
-                        {
-                            return (dynamic_cast<const ngraph::op::ConvolutionBiasAdd*>(node))
-                                ->with_relu();
-                        }
-                        if (dynamic_cast<const ngraph::op::ConvolutionAdd*>(node))
-                        {
-                            return (dynamic_cast<const ngraph::op::ConvolutionAdd*>(node))
-                                ->with_relu();
-                        }
-                        if (dynamic_cast<const ngraph::op::ConvolutionRelu*>(node))
-                        {
-                            return true;
-                        }
-                        if (dynamic_cast<const ngraph::op::QuantizedConvolutionRelu*>(node))
-                        {
-                            return true;
-                        }
-                        if (dynamic_cast<const ngraph::op::QuantizedConvolutionBias*>(node))
-                        {
-                            return (dynamic_cast<const ngraph::op::QuantizedConvolutionBias*>(node))
-                                ->with_relu();
-                        }
-                        if (dynamic_cast<const ngraph::op::QuantizedConvolutionBiasAdd*>(node))
-                        {
-                            return (dynamic_cast<const ngraph::op::QuantizedConvolutionBiasAdd*>(
-                                        node))
-                                ->with_relu();
-                        }
-                        if (dynamic_cast<const ngraph::op::QuantizedConvolutionBiasSignedAdd*>(
-                                node))
-                        {
-                            return (dynamic_cast<
-                                        const ngraph::op::QuantizedConvolutionBiasSignedAdd*>(node))
-                                ->with_relu();
-                        }
-
-                        return false;
-                    };
-
-                    if (add_relu())
+                    if (has_relu<OP>(node))
                     {
                         const float ops_scale = 1.f;
                         const float ops_alpha = -0.f; // relu negative slope
@@ -626,6 +583,244 @@ namespace ngraph
                 size_t build_quantize_reorder(const mkldnn::memory::desc& input_desc,
                                               const mkldnn::memory::desc& result_desc,
                                               const std::vector<float>& scales);
+                template <typename OP>
+                size_t get_scale_index()
+                {
+                    if (std::is_same<OP, ngraph::op::QuantizedConvolution>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionRelu>())
+                    {
+                        return 2;
+                    }
+                    if (std::is_same<OP, ngraph::op::QuantizedConvolutionBias>())
+                    {
+                        return 3;
+                    }
+                    if (std::is_same<OP, ngraph::op::QuantizedConvolutionBiasAdd>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBiasSignedAdd>())
+                    {
+                        return 4;
+                    }
+                }
+
+                template <typename OP, typename T>
+                std::vector<T> get_output_scale(const ngraph::Node* node)
+                {
+                    auto index = get_scale_index<OP>();
+                    std::vector<T> scale_val = {0};
+                    auto scale_const_op = std::dynamic_pointer_cast<ngraph::op::Constant>(
+                        node->get_arguments()[index]);
+                    if (scale_const_op != nullptr)
+                    {
+                        scale_val = scale_const_op->template get_vector<T>();
+                    }
+
+                    return scale_val;
+                }
+
+                template <typename OP,
+                          typename std::enable_if<
+                              (std::is_same<OP, ngraph::op::Convolution>::value ||
+                               std::is_same<OP, ngraph::op::QuantizedConvolution>::value),
+                              std::nullptr_t>::type = nullptr>
+                bool has_relu(const ngraph::Node* node)
+                {
+                    return false;
+                }
+
+                template <typename OP,
+                          typename std::enable_if<
+                              (!std::is_same<OP, ngraph::op::Convolution>::value &&
+                               !std::is_same<OP, ngraph::op::QuantizedConvolution>::value),
+                              std::nullptr_t>::type = nullptr>
+                bool has_relu(const ngraph::Node* node)
+                {
+                    return static_cast<const OP*>(node)->with_relu();
+                }
+
+                template <typename OP>
+                bool has_bias()
+                {
+                    if (std::is_same<OP, ngraph::op::ConvolutionBias>() ||
+                        std::is_same<OP, ngraph::op::ConvolutionBiasAdd>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBias>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBiasAdd>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBiasSignedAdd>())
+                    {
+                        return true;
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                }
+
+                template <typename OP>
+                bool is_quantized_conv()
+                {
+                    if (std::is_same<OP, ngraph::op::QuantizedConvolution>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionRelu>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBias>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBiasAdd>() ||
+                        std::is_same<OP, ngraph::op::QuantizedConvolutionBiasSignedAdd>())
+                    {
+                        return true;
+                    }
+                    else
+                    {
+                        return false;
+                    }
+                }
+
+                template <typename OP>
+                mkldnn::convolution_forward::desc
+                    get_convolution_forward_desc(const ngraph::Node* node,
+                                                 const std::vector<TensorViewWrapper>& args,
+                                                 const std::vector<TensorViewWrapper>& out)
+                {
+                    auto convolution = static_cast<const OP*>(node);
+                    // For dilation, MKLDNN wants to know how many elements to insert between, not how far
+                    // apart to space the elements like nGraph. So we have to subtract 1 from each pos.
+                    Strides window_dilation_strides_adjusted;
+
+                    for (size_t s : convolution->get_window_dilation_strides())
+                    {
+                        window_dilation_strides_adjusted.push_back(s - 1);
+                    }
+
+                    auto data_desc = mkldnn_utils::get_input_mkldnn_md(node, 0);
+                    auto weights_desc = mkldnn_utils::get_input_mkldnn_md(node, 1);
+                    // MKLDNN relies on named formats for kernel selection
+                    if (weights_desc.data.format == mkldnn_nchw)
+                        weights_desc.data.format = mkldnn_oihw;
+                    if (weights_desc.data.format == mkldnn_ncdhw)
+                        weights_desc.data.format = mkldnn_oidhw;
+                    auto result_desc = mkldnn_utils::get_output_mkldnn_md(node, 0);
+
+                    if (has_bias<OP>())
+                    {
+                        auto bias_desc = mkldnn_utils::get_input_mkldnn_md(node, 2);
+                        return mkldnn::convolution_forward::desc(
+                            mkldnn::prop_kind::forward,
+                            mkldnn::algorithm::convolution_direct,
+                            data_desc,
+                            weights_desc,
+                            bias_desc,
+                            result_desc,
+                            MKLDNN_DIMS(convolution->get_window_movement_strides()),
+                            MKLDNN_DIMS(window_dilation_strides_adjusted),
+                            MKLDNN_DIMS(convolution->get_padding_below()),
+                            MKLDNN_DIMS(convolution->get_padding_above()),
+                            mkldnn::padding_kind::zero);
+                    }
+                    else
+                    {
+                        return mkldnn::convolution_forward::desc(
+                            mkldnn::prop_kind::forward,
+                            mkldnn::algorithm::convolution_direct,
+                            data_desc,
+                            weights_desc,
+                            result_desc,
+                            MKLDNN_DIMS(convolution->get_window_movement_strides()),
+                            MKLDNN_DIMS(window_dilation_strides_adjusted),
+                            MKLDNN_DIMS(convolution->get_padding_below()),
+                            MKLDNN_DIMS(convolution->get_padding_above()),
+                            mkldnn::padding_kind::zero);
+                    }
+                }
+
+                template <typename OP>
+                mkldnn::primitive_attr get_convolution_forward_attr(const ngraph::Node* node)
+                {
+                    mkldnn::post_ops ops;
+
+                    if (std::is_same<OP, ngraph::op::ConvolutionBiasAdd>() ||
+                        std::is_same<OP, ngraph::op::ConvolutionAdd>())
+                    {
+                        ops.append_sum(1.f);
+                    }
+
+                    if (std::is_same<OP, ngraph::op::QuantizedConvolutionBiasAdd>())
+                    {
+                        auto sum_scale_val =
+                            extract_scale_value<ngraph::op::QuantizedConvolutionBiasAdd>(node, 5);
+                        ops.append_sum(sum_scale_val[0]);
+                    }
+
+                    if (std::is_same<OP, ngraph::op::QuantizedConvolutionBiasSignedAdd>())
+                    {
+                        auto sum_scale_val =
+                            extract_scale_value<ngraph::op::QuantizedConvolutionBiasSignedAdd>(node,
+                                                                                               5);
+                        ops.append_sum(2.0 * sum_scale_val[0]);
+                    }
+
+                    if (has_relu<OP>(node))
+                    {
+                        const float ops_scale = 1.f;
+                        const float ops_alpha = -0.f; // relu negative slope
+                        const float ops_beta = 0.f;
+                        ops.append_eltwise(
+                            ops_scale, mkldnn::algorithm::eltwise_relu, ops_alpha, ops_beta);
+                    }
+
+                    mkldnn::primitive_attr conv_attr;
+                    conv_attr.set_post_ops(ops);
+                    if (is_quantized_conv<OP>())
+                    {
+                        conv_attr.set_int_output_round_mode(mkldnn::round_mode::round_nearest);
+                        conv_attr.set_output_scales(0, get_output_scale<OP, float>(node));
+                    }
+                    return conv_attr;
+                }
+
+                size_t convolution_forward_init(bool with_bias = false);
+
+                template <bool with_bias>
+                void convolution_forward(const mkldnn::convolution_forward::desc& desc,
+                                         const mkldnn::primitive_attr& attr,
+                                         const mkldnn::engine& engine,
+                                         size_t& conv_idx)
+                {
+                    size_t input_idx, weights_idx, results_idx, bias_idx;
+                    input_idx = m_primitive_deps[conv_idx][0];
+                    weights_idx = m_primitive_deps[conv_idx][1];
+                    m_mkldnn_primitives[input_idx] =
+                        new mkldnn::memory({{desc.data.src_desc}, engine}, nullptr);
+                    m_mkldnn_primitives[weights_idx] =
+                        new mkldnn::memory({{desc.data.weights_desc}, engine}, nullptr);
+                    if (with_bias)
+                    {
+                        bias_idx = m_primitive_deps[conv_idx][2];
+                        results_idx = m_primitive_deps[conv_idx][3];
+                        m_mkldnn_primitives[bias_idx] =
+                            new mkldnn::memory({{desc.data.bias_desc}, engine}, nullptr);
+                    }
+                    else
+                    {
+                        results_idx = m_primitive_deps[conv_idx][2];
+                    }
+                    m_mkldnn_primitives[results_idx] =
+                        new mkldnn::memory({{desc.data.dst_desc}, engine}, nullptr);
+
+                    mkldnn::primitive* prim;
+                    if (with_bias)
+                    {
+                        prim = new mkldnn::convolution_forward({desc, attr, engine},
+                                                               *m_mkldnn_primitives[input_idx],
+                                                               *m_mkldnn_primitives[weights_idx],
+                                                               *m_mkldnn_primitives[bias_idx],
+                                                               *m_mkldnn_primitives[results_idx]);
+                    }
+                    else
+                    {
+                        prim = new mkldnn::convolution_forward({desc, attr, engine},
+                                                               *m_mkldnn_primitives[input_idx],
+                                                               *m_mkldnn_primitives[weights_idx],
+                                                               *m_mkldnn_primitives[results_idx]);
+                    }
+
+                    m_mkldnn_primitives[conv_idx] = prim;
+                }
 
             private:
                 std::vector<mkldnn::primitive*> m_mkldnn_primitives;

src/ngraph/runtime/cpu/op/conv_relu.hpp

@@ -45,6 +45,7 @@ namespace ngraph
             const Strides& get_data_dilation_strides() const { return m_data_dilation_strides; }
             std::shared_ptr<Node> get_filters() { return get_argument(1); }
             std::shared_ptr<Node> get_data_batch() { return get_argument(0); }
+            bool with_relu() const { return true; }
             virtual std::shared_ptr<Node>
                 copy_with_new_args(const NodeVector& new_args) const override;
 

test/builder_quantization.cpp

@@ -187,6 +187,59 @@ TEST(builder, scaled_QC)
               read_vector<int8_t>(result));
 }
 
+TEST(builder, dynamic_scaled_QC)
+{
+    Shape shape_a{1, 1, 3, 4}; // input shape
+    Shape shape_b{1, 1, 3, 3}; // filter shape
+    Shape shape_r{1, 1, 3, 4}; // output shape
+    vector<uint8_t> a_data = {1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4};
+    vector<int8_t> b_data = {1, 2, 3, 4, 5, 0, 0, 1, 2};
+    auto A = make_shared<op::Parameter>(element::u8, shape_a);
+    auto B = make_shared<op::Parameter>(element::i8, shape_b);
+    auto C = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto D = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto E = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto F = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto G = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto H = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto CV = ngraph::builder::ScaledQuantizedConvolution(A,
+                                                          B,
+                                                          Strides{1, 1},        // move_strides
+                                                          Strides{1, 1},        // filter_dilation
+                                                          CoordinateDiff{1, 1}, // below_pads
+                                                          CoordinateDiff{1, 1}, // above_pads
+                                                          Strides{1, 1},        // data_dilation
+                                                          C,
+                                                          D,
+                                                          E,
+                                                          F,
+                                                          G,
+                                                          H);
+    auto f = make_shared<Function>(NodeVector{CV}, ParameterVector{A, B, C, D, E, F, G, H});
+    auto backend = runtime::Backend::create("CPU");
+    // Create some tensors for input/output
+    auto a = backend->create_tensor(element::u8, shape_a);
+    copy_data(a, a_data);
+    auto b = backend->create_tensor(element::i8, shape_b);
+    copy_data(b, b_data);
+    auto d = backend->create_tensor(element::f32, Shape{1});
+    copy_data(d, vector<float>{0.0f});
+    auto e = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e, vector<float>{255.0f});
+    auto e_a = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e_a, vector<float>{-127.0f});
+    auto g = backend->create_tensor(element::f32, Shape{1});
+    copy_data(g, vector<float>{127.0f});
+    auto h = backend->create_tensor(element::f32, Shape{1});
+    copy_data(h, vector<float>{22.0f});
+    auto i = backend->create_tensor(element::f32, Shape{1});
+    copy_data(i, vector<float>{90.0f});
+    auto result = backend->create_tensor(element::i8, shape_r);
+    backend->call_with_validate(backend->compile(f), {result}, {a, b, d, e, e_a, g, h, i});
+    EXPECT_EQ((vector<int8_t>{31, 48, 42, 45, 54, 102, 127, 61, 47, 74, 61, 55}),
+              read_vector<int8_t>(result));
+}
+
 TEST(builder, scaled_QC_with_relu)
 {
     Shape shape_a{1, 1, 3, 3}; // input shape
@@ -229,6 +282,58 @@ TEST(builder, scaled_QC_with_relu)
     EXPECT_EQ((vector<uint8_t>{0, 0, 0, 0, 0, 0, 138, 212, 181}), read_vector<uint8_t>(result));
 }
 
+TEST(builder, dynamic_scaled_QC_with_relu)
+{
+    Shape shape_a{1, 1, 3, 3}; // input shape
+    Shape shape_b{1, 1, 3, 3}; // filter shape
+    Shape shape_r{1, 1, 3, 3}; // output shape
+    vector<uint8_t> a_data = {1, 2, 3, 4, 5, 6, 7, 8, 9};
+    vector<int8_t> b_data = {1, 2, 1, 0, 0, 0, -1, -2, -1};
+    auto A = make_shared<op::Parameter>(element::u8, shape_a);
+    auto B = make_shared<op::Parameter>(element::i8, shape_b);
+    auto C = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto D = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto E = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto F = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto G = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto H = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto CV = ngraph::builder::ScaledQuantizedConvolutionRelu(A,
+                                                              B,
+                                                              Strides{1, 1}, // move_strides
+                                                              Strides{1, 1}, // filter_dilation
+                                                              CoordinateDiff{1, 1}, // below_pads
+                                                              CoordinateDiff{1, 1}, // above_pads
+                                                              Strides{1, 1},        // data_dilation
+                                                              C,
+                                                              D,
+                                                              E,
+                                                              F,
+                                                              G,
+                                                              H);
+    auto f = make_shared<Function>(NodeVector{CV}, ParameterVector{A, B, C, D, E, F, G, H});
+    auto backend = runtime::Backend::create("CPU");
+    // Create some tensors for input/output
+    auto a = backend->create_tensor(element::u8, shape_a);
+    copy_data(a, a_data);
+    auto b = backend->create_tensor(element::i8, shape_b);
+    copy_data(b, b_data);
+    auto d = backend->create_tensor(element::f32, Shape{1});
+    copy_data(d, vector<float>{0.0f});
+    auto e = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e, vector<float>{255.0f});
+    auto e_a = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e_a, vector<float>{-127.0f});
+    auto g = backend->create_tensor(element::f32, Shape{1});
+    copy_data(g, vector<float>{127.0f});
+    auto h = backend->create_tensor(element::f32, Shape{1});
+    copy_data(h, vector<float>{20.0f});
+    auto i = backend->create_tensor(element::f32, Shape{1});
+    copy_data(i, vector<float>{-24.0f});
+    auto result = backend->create_tensor(element::u8, shape_r);
+    backend->call_with_validate(backend->compile(f), {result}, {a, b, d, e, e_a, g, h, i});
+    EXPECT_EQ((vector<uint8_t>{0, 0, 0, 0, 0, 0, 138, 212, 181}), read_vector<uint8_t>(result));
+}
+
 TEST(builder, scaled_QC_with_bias)
 {
     Shape shape_a{1, 1, 3, 4}; // input shape
@@ -277,6 +382,64 @@ TEST(builder, scaled_QC_with_bias)
               read_vector<int8_t>(result));
 }
 
+TEST(builder, dynamic_scaled_QC_with_bias)
+{
+    Shape shape_a{1, 1, 3, 4}; // input shape
+    Shape shape_b{1, 1, 3, 3}; // filter shape
+    Shape shape_r{1, 1, 3, 4}; // output shape
+    vector<uint8_t> a_data = {1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4};
+    vector<int8_t> b_data = {1, 2, 3, 4, 5, 0, 0, 1, 2};
+    vector<int32_t> c_data = {5};
+    auto A = make_shared<op::Parameter>(element::u8, shape_a);
+    auto B = make_shared<op::Parameter>(element::i8, shape_b);
+    auto Bias = make_shared<op::Parameter>(element::i32, Shape{1});
+    auto C = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto D = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto E = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto F = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto G = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto H = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto CV = ngraph::builder::ScaledQuantizedConvolutionBias(A,
+                                                              B,
+                                                              Bias,
+                                                              Strides{1, 1}, // move_strides
+                                                              Strides{1, 1}, // filter_dilation
+                                                              CoordinateDiff{1, 1}, // below_pads
+                                                              CoordinateDiff{1, 1}, // above_pads
+                                                              Strides{1, 1},        // data_dilation
+                                                              C,
+                                                              D,
+                                                              E,
+                                                              F,
+                                                              G,
+                                                              H);
+    auto f = make_shared<Function>(NodeVector{CV}, ParameterVector{A, B, Bias, C, D, E, F, G, H});
+    auto backend = runtime::Backend::create("CPU");
+    // Create some tensors for input/output
+    auto a = backend->create_tensor(element::u8, shape_a);
+    copy_data(a, a_data);
+    auto b = backend->create_tensor(element::i8, shape_b);
+    copy_data(b, b_data);
+    auto c = backend->create_tensor(element::i32, Shape{1});
+    copy_data(c, c_data);
+    auto d = backend->create_tensor(element::f32, Shape{1});
+    copy_data(d, vector<float>{0.0f});
+    auto e = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e, vector<float>{255.0f});
+    auto e_a = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e_a, vector<float>{-127.0f});
+    auto g = backend->create_tensor(element::f32, Shape{1});
+    copy_data(g, vector<float>{127.0f});
+    auto h = backend->create_tensor(element::f32, Shape{1});
+    copy_data(h, vector<float>{22.0f});
+    auto i = backend->create_tensor(element::f32, Shape{1});
+    copy_data(i, vector<float>{90.0f});
+    auto result = backend->create_tensor(element::i8, shape_r);
+    backend->call_with_validate(backend->compile(f), {result}, {a, b, c, d, e, e_a, g, h, i});
+    EXPECT_EQ((vector<int8_t>{38, 55, 50, 52, 61, 109, 127, 68, 54, 81, 68, 62}),
+              read_vector<int8_t>(result));
+}
+
 TEST(builder, scaled_QC_with_bias_and_relu)
 {
     Shape shape_a{1, 1, 3, 3}; // input shape
@@ -383,6 +546,80 @@ TEST(builder, scaled_QC_with_bias_add_and_relu)
               read_vector<uint8_t>(result));
 }
 
+TEST(builder, dynamic_scaled_QC_with_bias_add_and_relu)
+{
+    Shape shape_a{1, 1, 3, 4}; // input shape
+    Shape shape_b{1, 1, 3, 3}; // filter shape
+    Shape shape_r{1, 1, 3, 4}; // output shape
+    vector<uint8_t> a_data = {1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4};
+    vector<int8_t> b_data = {1, 2, 3, 4, 5, 0, 0, 1, 2};
+    vector<int32_t> c_data = {5};
+    vector<uint8_t> conv_2_data = {1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4};
+    auto A = make_shared<op::Parameter>(element::u8, shape_a);
+    auto B = make_shared<op::Parameter>(element::i8, shape_b);
+    auto Add = make_shared<op::Parameter>(element::u8, shape_a);
+    auto Bias = make_shared<op::Parameter>(element::i32, Shape{1});
+    auto C = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto D = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto E = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto F = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto G = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto H = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto I = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto J = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto CV = ngraph::builder::ScaledQuantizedConvolutionBiasAdd(A,
+                                                                 B,
+                                                                 Bias,
+                                                                 Add,
+                                                                 Strides{1, 1}, // move_strides
+                                                                 Strides{1, 1}, // filter_dilation
+                                                                 CoordinateDiff{1, 1}, // below_pads
+                                                                 CoordinateDiff{1, 1}, // above_pads
+                                                                 Strides{1, 1}, // data_dilation
+                                                                 C,
+                                                                 D,
+                                                                 E,
+                                                                 F,
+                                                                 G,
+                                                                 H,
+                                                                 I,
+                                                                 J,
+                                                                 true);
+    auto f = make_shared<Function>(NodeVector{CV},
+                                   ParameterVector{A, B, Bias, Add, C, D, E, F, G, H, I, J});
+    auto backend = runtime::Backend::create("CPU");
+    // Create some tensors for input/output
+    auto a = backend->create_tensor(element::u8, shape_a);
+    copy_data(a, a_data);
+    auto b = backend->create_tensor(element::i8, shape_b);
+    copy_data(b, b_data);
+    auto c = backend->create_tensor(element::i32, Shape{1});
+    copy_data(c, c_data);
+    auto d = backend->create_tensor(element::u8, shape_a);
+    copy_data(d, conv_2_data);
+    auto e = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e, vector<float>{0.0f});
+    auto e_a = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e_a, vector<float>{255.0f});
+    auto g = backend->create_tensor(element::f32, Shape{1});
+    copy_data(g, vector<float>{-127.0f});
+    auto h = backend->create_tensor(element::f32, Shape{1});
+    copy_data(h, vector<float>{127.0f});
+    auto i = backend->create_tensor(element::f32, Shape{1});
+    copy_data(i, vector<float>{22.0f});
+    auto j = backend->create_tensor(element::f32, Shape{1});
+    copy_data(j, vector<float>{90.0f});
+    auto k = backend->create_tensor(element::f32, Shape{1});
+    copy_data(k, vector<float>{22.0f});
+    auto l = backend->create_tensor(element::f32, Shape{1});
+    copy_data(l, vector<float>{180.0f});
+    auto result = backend->create_tensor(element::u8, shape_r);
+    auto handle = backend->compile(f);
+    backend->call_with_validate(handle, {result}, {a, b, c, d, e, e_a, g, h, i, j, k, l});
+    EXPECT_EQ((vector<uint8_t>{78, 114, 105, 113, 132, 230, 255, 136, 110, 165, 142, 133}),
+              read_vector<uint8_t>(result));
+}
+
 TEST(builder, scaled_QC_with_bias_signed_add_and_relu)
 {
     Shape shape_a{1, 1, 3, 4}; // input shape
@@ -442,6 +679,81 @@ TEST(builder, scaled_QC_with_bias_signed_add_and_relu)
               read_vector<uint8_t>(result));
 }
 
+TEST(builder, dynamic_scaled_QC_with_bias_signed_add_and_relu)
+{
+    Shape shape_a{1, 1, 3, 4}; // input shape
+    Shape shape_b{1, 1, 3, 3}; // filter shape
+    Shape shape_r{1, 1, 3, 4}; // output shape
+    vector<uint8_t> a_data = {1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4};
+    vector<int8_t> b_data = {1, 2, 3, 4, 5, 0, 0, 1, 2};
+    vector<int32_t> c_data = {5};
+    vector<int8_t> conv_2_data = {-1, -2, -3, -4, -5, -6, -10, 0, 1, 2, 3, 4};
+    auto A = make_shared<op::Parameter>(element::u8, shape_a);
+    auto B = make_shared<op::Parameter>(element::i8, shape_b);
+    auto Add = make_shared<op::Parameter>(element::i8, shape_a);
+    auto Bias = make_shared<op::Parameter>(element::i32, Shape{1});
+    auto C = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto D = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto E = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto F = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto G = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto H = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto I = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto J = make_shared<op::Parameter>(element::f32, Shape{1});
+    auto CV =
+        ngraph::builder::ScaledQuantizedConvolutionBiasSignedAdd(A,
+                                                                 B,
+                                                                 Bias,
+                                                                 Add,
+                                                                 Strides{1, 1}, // move_strides
+                                                                 Strides{1, 1}, // filter_dilation
+                                                                 CoordinateDiff{1, 1}, // below_pads
+                                                                 CoordinateDiff{1, 1}, // above_pads
+                                                                 Strides{1, 1}, // data_dilation
+                                                                 C,
+                                                                 D,
+                                                                 E,
+                                                                 F,
+                                                                 G,
+                                                                 H,
+                                                                 I,
+                                                                 J,
+                                                                 true);
+    auto f = make_shared<Function>(NodeVector{CV},
+                                   ParameterVector{A, B, Bias, Add, C, D, E, F, G, H, I, J});
+    auto backend = runtime::Backend::create("CPU");
+    // Create some tensors for input/output
+    auto a = backend->create_tensor(element::u8, shape_a);
+    copy_data(a, a_data);
+    auto b = backend->create_tensor(element::i8, shape_b);
+    copy_data(b, b_data);
+    auto c = backend->create_tensor(element::i32, Shape{1});
+    copy_data(c, c_data);
+    auto d = backend->create_tensor(element::i8, shape_a);
+    copy_data(d, conv_2_data);
+    auto e = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e, vector<float>{0.0f});
+    auto e_a = backend->create_tensor(element::f32, Shape{1});
+    copy_data(e_a, vector<float>{255.0f});
+    auto g = backend->create_tensor(element::f32, Shape{1});
+    copy_data(g, vector<float>{-127.0f});
+    auto h = backend->create_tensor(element::f32, Shape{1});
+    copy_data(h, vector<float>{127.0f});
+    auto i = backend->create_tensor(element::f32, Shape{1});
+    copy_data(i, vector<float>{22.0f});
+    auto j = backend->create_tensor(element::f32, Shape{1});
+    copy_data(j, vector<float>{90.0f});
+    auto k = backend->create_tensor(element::f32, Shape{1});
+    copy_data(k, vector<float>{22.0f});
+    auto l = backend->create_tensor(element::f32, Shape{1});
+    copy_data(l, vector<float>{90.0f});
+    auto result = backend->create_tensor(element::u8, shape_r);
+    auto handle = backend->compile(f);
+    backend->call_with_validate(handle, {result}, {a, b, c, d, e, e_a, g, h, i, j, k, l});
+    EXPECT_EQ((vector<uint8_t>{76, 110, 99, 105, 122, 218, 255, 136, 110, 165, 142, 133}),
+              read_vector<uint8_t>(result));
+}
+
 TEST(builder, scaled_QC_with_f32_bias_and_relu)
 {
     Shape shape_a{1, 1, 3, 3}; // input shape
@@ -512,29 +824,72 @@ TEST(builder, scaled_Q_unsigned)
     EXPECT_EQ((vector<uint8_t>{0, 0, 1, 1, 2, 64, 127, 255}), read_vector<uint8_t>(result));
 }
 
-TEST(builder, dynamic_scaled_Q_unsigned)
+TEST(builder, dynamic_scaled_Q)
 {
-    vector<float> a_data = {-255.0, 0.0, 1.0, 1.25, 1.75, 64.0, 127.0, 500.0};
-    Shape shape_a{8};
-    AxisSet quantization_axes;
-    op::Quantize::RoundMode round_mode = op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN;
-    auto A = make_shared<op::Parameter>(element::f32, shape_a);
-    auto B = make_shared<op::Parameter>(element::f32, Shape{});
-    auto C = make_shared<op::Parameter>(element::f32, Shape{});
-    auto QT = ngraph::builder::ScaledQuantize(A, B, C, element::u8, quantization_axes, round_mode);
-    auto f = make_shared<Function>(NodeVector{QT}, ParameterVector{A, B, C});
+    auto call_SQ = [](unique_ptr<runtime::Backend>& backend,
+                      element::Type type,
+                      op::Quantize::RoundMode mode,
+                      Shape in_shape,
+                      vector<float> in,
+                      float min,
+                      float max) {
+        auto A = make_shared<op::Parameter>(element::f32, in_shape);
+        auto B = make_shared<op::Parameter>(element::f32, Shape{});
+        auto C = make_shared<op::Parameter>(element::f32, Shape{});
+        auto QT = ngraph::builder::ScaledQuantize(A, B, C, type, AxisSet{}, mode);
+        auto f = make_shared<Function>(NodeVector{QT}, ParameterVector{A, B, C});
+        // Create some tensors for input/output
+        auto a = backend->create_tensor(element::f32, in_shape);
+        auto b = backend->create_tensor(element::f32, Shape{});
+        auto c = backend->create_tensor(element::f32, Shape{});
+        copy_data(a, in);
+        copy_data(b, vector<float>{min});
+        copy_data(c, vector<float>{max});
+        auto result = backend->create_tensor(type, in_shape);
+        backend->call_with_validate(backend->compile(f), {result}, {a, b, c});
+        return result;
+    };
     auto backend = runtime::Backend::create("CPU");
-    // Create some tensors for input/output
-    auto a = backend->create_tensor(element::f32, shape_a);
-    auto b = backend->create_tensor(element::f32, Shape{});
-    auto c = backend->create_tensor(element::f32, Shape{});
-    copy_data(a, a_data);
-    copy_data(b, vector<float>{-255.0f});
-    copy_data(c, vector<float>{127.0f});
-    auto result = backend->create_tensor(element::u8, shape_a);
-    auto handle = backend->compile(f);
-    backend->call_with_validate(handle, {result}, {a, b, c});
+    auto result = call_SQ(backend,
+                          element::u8,
+                          op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN,
+                          Shape{8},
+                          vector<float>{-255.0, 0.0, 1.0, 1.25, 1.75, 64.0, 127.0, 500.0},
+                          -255.0f,
+                          127.0f);
     EXPECT_EQ((vector<uint8_t>{0, 0, 1, 1, 2, 64, 127, 255}), read_vector<uint8_t>(result));
+    auto result2 = call_SQ(backend,
+                           element::u8,
+                           op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN,
+                           Shape{8},
+                           vector<float>{-85.0, 0.0, 2.0, 10.0, 15.0, 127.0, 64.0, 500.0},
+                           -85.0f,
+                           15.0f);
+    EXPECT_EQ((vector<uint8_t>{0, 0, 6, 30, 45, 255, 192, 255}), read_vector<uint8_t>(result2));
+    auto result3 = call_SQ(backend,
+                           element::u8,
+                           op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN,
+                           Shape{2, 2},
+                           vector<float>{0.1392, 0.5928, 0.6027, 0.8579},
+                           0.0f,
+                           1.0f);
+    EXPECT_EQ((vector<uint8_t>{35, 151, 154, 219}), read_vector<uint8_t>(result3));
+    auto result4 = call_SQ(backend,
+                           element::i8,
+                           op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN,
+                           Shape{2, 4},
+                           vector<float>{-1.3990955,
+                                         -1.468798,
+                                         -2.0760186,
+                                         0.17088544,
+                                         -0.0829789,
+                                         -0.3173087,
+                                         -0.5645172,
+                                         -0.3188769},
+                           -2.0760186f,
+                           0.17088544f);
+    EXPECT_EQ((vector<int8_t>{-86, -90, -127, 10, -5, -19, -35, -20}),
+              read_vector<int8_t>(result4));
 }
 
 TEST(builder, scaled_Q_signed)
@@ -578,3 +933,39 @@ TEST(builder, scaled_DQ_signed)
     backend->call_with_validate(handle, {result}, {a});
     EXPECT_EQ((vector<float>{99.212601}), read_vector<float>(result));
 }
+
+template <typename T>
+shared_ptr<runtime::Tensor> call_SDQ(unique_ptr<runtime::Backend>& backend,
+                                     element::Type type,
+                                     Shape in_shape,
+                                     vector<T> in,
+                                     float min,
+                                     float max)
+{
+    auto A = make_shared<op::Parameter>(type, in_shape);
+    auto B = make_shared<op::Parameter>(element::f32, Shape{});
+    auto C = make_shared<op::Parameter>(element::f32, Shape{});
+    auto DQT = ngraph::builder::ScaledDequantize(A, B, C, element::f32, AxisSet{});
+    auto f = make_shared<Function>(NodeVector{DQT}, ParameterVector{A, B, C});
+    // Create some tensors for input/output
+    auto a = backend->create_tensor(type, in_shape);
+    auto b = backend->create_tensor(element::f32, Shape{});
+    auto c = backend->create_tensor(element::f32, Shape{});
+    copy_data(a, in);
+    copy_data(b, vector<float>{min});
+    copy_data(c, vector<float>{max});
+    auto result = backend->create_tensor(element::f32, in_shape);
+    backend->call_with_validate(backend->compile(f), {result}, {a, b, c});
+    return result;
+}
+TEST(builder, dynamic_scaled_DQ)
+{
+    auto backend = runtime::Backend::create("CPU");
+    auto result =
+        call_SDQ<int8_t>(backend, element::i8, Shape{1}, vector<int8_t>{42}, -1.0f, 300.0f);
+    EXPECT_EQ((vector<float>{99.212601}), read_vector<float>(result));
+    auto result2 = call_SDQ<uint8_t>(
+        backend, element::u8, Shape{2, 2}, vector<uint8_t>{35, 151, 154, 219}, 0.0f, 1.0f);
+    EXPECT_EQ((vector<float>{0.13725491, 0.59215689, 0.60392159, 0.8588236}),
+              read_vector<float>(result2));
+}