Divert dynamic offset to reference (#2339)

* Divert dynamic offset to reference

* update gpu manifest

* Avoid dynamic cast

src/ngraph/runtime/cpu/pass/cpu_assignment.cpp

@@ -728,6 +728,12 @@ namespace ngraph
                 void CPUAssignment::ASSIGN_DECL(ngraph::op::Dequantize)
                 {
                     auto dequantize = static_cast<op::Dequantize*>(node);
+                    // TODO(nbpatel): Support dynamic offset via mkldnn
+                    // Go through reference if the offset is not a constant
+                    if (!dequantize->get_argument(2)->is_constant())
+                    {
+                        return;
+                    }
                     auto offset_const_op =
                         std::static_pointer_cast<ngraph::op::Constant>(dequantize->get_argument(2));
                     // TODO: MKLDNN only handles float / not double
@@ -760,6 +766,12 @@ namespace ngraph
                 void CPUAssignment::ASSIGN_DECL(ngraph::op::Quantize)
                 {
                     auto quantize = static_cast<op::Quantize*>(node);
+                    // TODO(nbpatel): Support dynamic offset via mkldnn
+                    // Go through reference if the offset is not a constant
+                    if (!quantize->get_argument(2)->is_constant())
+                    {
+                        return;
+                    }
                     auto offset_const_op =
                         std::static_pointer_cast<ngraph::op::Constant>(quantize->get_argument(2));
                     op::Quantize::RoundMode round_mode = quantize->get_round_mode();

src/ngraph/runtime/gpu/unit_test.manifest

@@ -26,6 +26,7 @@ backwards_batch_norm_training
 dequantize
 dequantize_zero_offset
 dequantize_axes
+dequantize_dynamic_offset
 dequantize_int8
 dequantize_int8_zero_offset
 dequantize_int32
@@ -33,6 +34,7 @@ dequantize_int32_zero_offset
 quantize
 quantize_zero_offset
 quantize_axes
+quantize_dynamic_offset
 quantize_int8
 quantize_int8_zero_offset
 quantize_int32

src/ngraph/runtime/intelgpu/unit_test.manifest

@@ -23,6 +23,7 @@ batch_norm_three_outputs
 batch_norm_bprop_n4c3h2w2
 dequantize
 dequantize_axes
+dequantize_dynamic_offset
 dequantize_int32
 dequantize_int32_zero_offset
 dequantize_int8
@@ -41,6 +42,7 @@ quantize_axes
 quantize_clamp_int32
 quantize_clamp_int8
 quantize_clamp_uint8
+quantize_dynamic_offset
 quantize_int32
 quantize_int32_zero_offset
 quantize_int8

src/ngraph/runtime/plaidml/unit_test.manifest

@@ -60,6 +60,7 @@ maxpool_bprop_larger_than_cache
 generate_mask
 avg_pool_3d
 avg_pool_3d_uneven_strided_padded_include_in_computation
+quantize_dynamic_offset                 # Quantization/Dequantization is unimplemented
 dequantize_int8_zero_offset             # Quantization/Dequantization is unimplemented
 dequantize_int32                        # Quantization/Dequantization is unimplemented
 dequantize_int32_zero_offset            # Quantization/Dequantization is unimplemented
@@ -76,6 +77,7 @@ quantize_ROUND_DOWN                     # Quantization/Dequantization is unimple
 quantize                                # Quantization/Dequantization is unimplemented
 quantize_zero_offset                    # Quantization/Dequantization is unimplemented
 quantize_axes                           # Quantization/Dequantization is unimplemented
+quantize_dynamic_offset                 # Quantization/Dequantization is unimplemented
 quantize_int8                           # Quantization/Dequantization is unimplemented
 quantize_int8_zero_offset               # Quantization/Dequantization is unimplemented
 quantize_int32                          # Quantization/Dequantization is unimplemented

test/backend_test.in.cpp

@@ -6628,3 +6628,78 @@ NGRAPH_TEST(${BACKEND_NAME}, shape_of_5d)
     vector<uint64_t> expected{2, 4, 8, 16, 32};
     EXPECT_EQ(expected, read_vector<uint64_t>(result));
 }
+
+NGRAPH_TEST(${BACKEND_NAME}, dequantize_dynamic_offset)
+{
+    Shape input_shape{4};
+    Shape scale_offset_shape = {};
+    AxisSet quantization_axes;
+
+    auto input_type = element::u8;
+    auto output_type = element::f32;
+
+    typedef uint8_t input_c_type;
+    typedef float output_c_type;
+
+    auto X = make_shared<op::Parameter>(input_type, input_shape);
+    auto scale = make_shared<op::Parameter>(output_type, scale_offset_shape);
+    auto offset = make_shared<op::Parameter>(input_type, scale_offset_shape);
+    auto dequantize = make_shared<op::Dequantize>(X, scale, offset, output_type, quantization_axes);
+    auto f = make_shared<Function>(dequantize, ParameterVector{X, scale, offset});
+
+    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    auto x = backend->create_tensor(input_type, input_shape);
+    auto y = backend->create_tensor(output_type, input_shape);
+    auto Scale = backend->create_tensor(output_type, scale_offset_shape);
+    auto Offset = backend->create_tensor(input_type, scale_offset_shape);
+
+    copy_data(x, vector<input_c_type>{0, 3, 128, 255});
+    copy_data(Scale, vector<output_c_type>{2});
+    copy_data(Offset, vector<input_c_type>{128});
+
+    auto handle = backend->compile(f);
+    backend->call_with_validate(handle, {y}, {x, Scale, Offset});
+    EXPECT_EQ((vector<output_c_type>{-256.0f, -250.0f, 0.0f, 254.0f}),
+              read_vector<output_c_type>(y));
+}
+
+NGRAPH_TEST(${BACKEND_NAME}, quantize_dynamic_offset)
+{
+    Shape input_shape{4, 3};
+    Shape scale_offset_shape = {};
+    AxisSet quantization_axes;
+
+    auto input_type = element::f32;
+    auto output_type = element::u8;
+
+    typedef float input_c_type;
+    typedef uint8_t output_c_type;
+
+    op::Quantize::RoundMode round_mode = op::Quantize::RoundMode::ROUND_NEAREST_TOWARD_EVEN;
+
+    auto X = make_shared<op::Parameter>(input_type, input_shape);
+    auto scale = make_shared<op::Parameter>(input_type, scale_offset_shape);
+    auto offset = make_shared<op::Parameter>(output_type, scale_offset_shape);
+    auto quantize =
+        make_shared<op::Quantize>(X, scale, offset, output_type, quantization_axes, round_mode);
+    auto f = make_shared<Function>(quantize, ParameterVector{X, scale, offset});
+
+    auto backend = runtime::Backend::create("${BACKEND_NAME}");
+    auto x = backend->create_tensor(input_type, input_shape);
+    auto y = backend->create_tensor(output_type, input_shape);
+    auto Scale = backend->create_tensor(input_type, scale_offset_shape);
+    auto Offset = backend->create_tensor(output_type, scale_offset_shape);
+
+    copy_data(x, vector<input_c_type>{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11});
+    // divide by scale                2  2  2  2  2  2  2  2  2  2  2   2
+    // equals (rounded)               0  0  1  2  2  2  3  4  4  4  5   6
+    // plus offset                    1  1  1  1  1  1  1  1  1  1  1   1
+    // equals                         1  1  2  3  3  3  4  5  5  5  6   7
+    copy_data(Scale, vector<input_c_type>{2});
+    copy_data(Offset, vector<output_c_type>{1});
+
+    auto handle = backend->compile(f);
+    backend->call_with_validate(handle, {y}, {x, Scale, Offset});
+    EXPECT_EQ((vector<output_c_type>{1, 1, 2, 3, 3, 3, 4, 5, 5, 5, 6, 7}),
+              read_vector<output_c_type>(y));
+}