Skip to content

N
ngraph
Commit 8da348f5 authored by Michał Karzyński's avatar Michał Karzyński Committed by Sang Ik Lee
Browse files
Options

[ONNX] Refactor quantized ops tests (#2798) 

* Add NgraphTestCase test runner class

* Code improvements

* clang-format

* Add support for multiple output types

* Add reading inputs/outputs from files

* Refactor model_quant_conv_linear_3d

* Add shape checking to NgraphTestCase

* Add expected output shape to model_quant_conv_linear_3d

* Code review

* Remove small data files and move values to test code for legibility

* clang-format

* Refactor model_quant_conv_linear_2d

* Refactor model_dequantize_linear_1d_zero_scale_uint8_negative_axis

* Refactor model_dequantize_linear_1d_zero_scale_int8_4d

* Refactor model_dequantize_linear_1d_zero_scale_int8

* Refactor model_dequantize_linear_1d_zero_scale_uint8

* Refactor model_dequantize_linear_scalar_zero_scale_int8

* Refactor model_dequantize_linear_scalar_zero_scale_uint8

* Refactor model_quantize_linear_axis_negative

* Refactor model_quantize_linear_axis_zero

* Refactor model_quantize_linear_zero_point

* Add shape checking to NgraphTestCase::run

* Refactor NgraphTestCase::run
parent d9dfd6c4
Hide whitespace changes
Inline Side-by-side
Showing with 181 additions and 427 deletions
test/files/onnx/qlinearconv2d/x_scale.bin deleted 100644 → 0
View file @ d9dfd6c4
--broken encoding: IBM424_ltr
\ No newline at end of file
test/onnx/onnx_import_quant.in.cpp
View file @ 8da348f5
...@@ -30,6 +30,7 @@ ...@@ -30,6 +30,7 @@
#include "util/all_close.hpp" #include "util/all_close.hpp"
#include "util/all_close_f.hpp" #include "util/all_close_f.hpp"
#include "util/ndarray.hpp" #include "util/ndarray.hpp"
#include "util/test_case.hpp"
#include "util/test_control.hpp" #include "util/test_control.hpp"
#include "util/test_tools.hpp" #include "util/test_tools.hpp"
...@@ -62,19 +63,14 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_zero_point) ...@@ -62,19 +63,14 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_zero_point)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/quantize_linear_zero_point.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/quantize_linear_zero_point.prototxt"));
Inputs inputs; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
inputs.emplace_back(std::vector<float>{0.f, 2.f, 3.f, 1000.f, -254.f, -1000.f}); // x test_case.add_input(std::vector<float>{0.f, 2.f, 3.f, 1000.f, -254.f, -1000.f}); // x
inputs.emplace_back(std::vector<float>{2.0f}); // y_scale test_case.add_input(std::vector<float>{2.0f}); // y_scale
test_case.add_input(std::vector<std::uint8_t>{128}); // y_zero_point
std::vector<std::vector<std::uint8_t>> int_inputs;
int_inputs.emplace_back(std::vector<std::uint8_t>{128}); // y_zero_point
std::vector<std::vector<std::uint8_t>> expected_output{
std::vector<std::uint8_t>{128, 129, 130, 255, 1, 0}};
std::vector<std::vector<std::uint8_t>> outputs{execute<float, std::uint8_t, std::uint8_t>( test_case.add_expected_output<std::uint8_t>(
function, inputs, int_inputs, "${BACKEND_NAME}")}; {6}, std::vector<std::uint8_t>{128, 129, 130, 255, 1, 0});
EXPECT_TRUE(test::all_close(expected_output.front(), outputs.front())); test_case.run();
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_zero) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_zero)
...@@ -82,21 +78,16 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_zero) ...@@ -82,21 +78,16 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_zero)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/quantize_linear_axis_zero.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/quantize_linear_axis_zero.prototxt"));
Inputs inputs; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
inputs.emplace_back(std::vector<float>{ test_case.add_input(std::vector<float>{
0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f}); // x 0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f}); // x
inputs.emplace_back(std::vector<float>{1.f, 2.f, 4.f}); // y_scale test_case.add_input(std::vector<float>{1.f, 2.f, 4.f}); // y_scale
test_case.add_input(std::vector<std::uint8_t>{0, 0, 0}); // y_zero_point
std::vector<std::vector<std::uint8_t>> int_inputs;
int_inputs.emplace_back(std::vector<std::uint8_t>{0, 0, 0}); // y_zero_point
std::vector<std::vector<std::uint8_t>> expected_output{ // std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 1, 1, 250}}; <- bad expected output given HALF_TO_EVEN round mode
// std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 1, 1, 250}}; <- bad expected output given HALF_TO_EVEN round mode test_case.add_expected_output<std::uint8_t>(
std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250}}; {3, 4}, std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250});
test_case.run();
std::vector<std::vector<std::uint8_t>> outputs{execute<float, std::uint8_t, std::uint8_t>(
function, inputs, int_inputs, "${BACKEND_NAME}")};
EXPECT_EQ(expected_output.front(), outputs.front());
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_negative) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_negative)
...@@ -104,21 +95,16 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_negative) ...@@ -104,21 +95,16 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quantize_linear_axis_negative)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/quantize_linear_axis_negative.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/quantize_linear_axis_negative.prototxt"));
Inputs inputs; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
inputs.emplace_back(std::vector<float>{ test_case.add_input(std::vector<float>{
0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f}); // x 0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f, 0.f, 2.f, 3.f, 1000.f}); // x
inputs.emplace_back(std::vector<float>{1.f, 2.f, 4.f}); // y_scale test_case.add_input(std::vector<float>{1.f, 2.f, 4.f}); // y_scale
test_case.add_input(std::vector<std::uint8_t>{0, 0, 0}); // y_zero_point
std::vector<std::vector<std::uint8_t>> int_inputs;
int_inputs.emplace_back(std::vector<std::uint8_t>{0, 0, 0}); // y_zero_point
std::vector<std::vector<std::uint8_t>> expected_output{ // std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 1, 1, 250}}; <- bad expected output given HALF_TO_EVEN round mode
// std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 1, 1, 250}}; <- bad expected output given HALF_TO_EVEN round mode test_case.add_expected_output<std::uint8_t>(
std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250}}; {3, 4}, std::vector<std::uint8_t>{0, 2, 3, 255, 0, 1, 2, 255, 0, 0, 1, 250});
test_case.run();
std::vector<std::vector<std::uint8_t>> outputs{execute<float, std::uint8_t, std::uint8_t>(
function, inputs, int_inputs, "${BACKEND_NAME}")};
EXPECT_TRUE(test::all_close(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear)
...@@ -140,38 +126,13 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_scalar_zero_scale_uint ...@@ -140,38 +126,13 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_scalar_zero_scale_uint
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_0.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_0.prototxt"));
auto x = std::vector<uint8_t>{0, 3, 128, 255}; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
auto scale = std::vector<float>{2.0f}; test_case.add_input(std::vector<uint8_t>{0, 3, 128, 255}); // x
auto zero_point = std::vector<uint8_t>{128}; test_case.add_input(std::vector<float>{2.0f}); // scale
test_case.add_input(std::vector<uint8_t>{128}); // zero_point
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}"); test_case.add_expected_output<float>({4}, std::vector<float>{-256.0f, -250.0f, 0.0f, 254.0f});
test_case.run();
auto params = function->get_parameters();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors;
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
copy_data(input_tensors[0], x);
copy_data(input_tensors[1], scale);
copy_data(input_tensors[2], zero_point);
auto results = function->get_results();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors;
result_tensors.push_back(
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape()));
auto handle = backend->compile(function);
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<float>> outputs;
outputs.push_back(read_vector<float>(result_tensors[0]));
auto expected_output = std::vector<std::vector<float>>{{-256.0f, -250.0f, 0.0f, 254.0f}};
EXPECT_TRUE(test::all_close_f(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_scalar_zero_scale_int8) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_scalar_zero_scale_int8)
...@@ -179,38 +140,14 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_scalar_zero_scale_int8 ...@@ -179,38 +140,14 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_scalar_zero_scale_int8
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_1.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_1.prototxt"));
auto x = std::vector<int8_t>{-30, -3, 100, 127}; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
auto scale = std::vector<float>{2.0f};
auto zero_point = std::vector<int8_t>{-10};
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}"); test_case.add_input(std::vector<int8_t>{-30, -3, 100, 127}); // x
test_case.add_input(std::vector<float>{2.0f}); // scale
test_case.add_input(std::vector<int8_t>{-10}); // zero_point
auto params = function->get_parameters(); test_case.add_expected_output<float>({4}, std::vector<float>{-40.0f, 14.0f, 220.0f, 274.0f});
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors; test_case.run();
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
copy_data(input_tensors[0], x);
copy_data(input_tensors[1], scale);
copy_data(input_tensors[2], zero_point);
auto results = function->get_results();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors;
result_tensors.push_back(
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape()));
auto handle = backend->compile(function);
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<float>> outputs;
outputs.push_back(read_vector<float>(result_tensors[0]));
auto expected_output = std::vector<std::vector<float>>{{-40.0f, 14.0f, 220.0f, 274.0f}};
EXPECT_TRUE(test::all_close_f(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8)
...@@ -218,39 +155,17 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8) ...@@ -218,39 +155,17 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_2.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_2.prototxt"));
auto x = std::vector<uint8_t>{0, 1, 2, 3, 0, 1, 2, 3, 0, 10, 20, 30}; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
auto scale = std::vector<float>{1.0f, 2.0f, 4.0f};
auto zero_point = std::vector<uint8_t>{0, 0, 0};
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}");
auto params = function->get_parameters(); test_case.add_input(std::vector<uint8_t>{0, 1, 2, 3, 0, 1, 2, 3, 0, 10, 20, 30}); // x
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors; test_case.add_input(std::vector<float>{1.0f, 2.0f, 4.0f}); // scale
input_tensors.push_back( test_case.add_input(std::vector<uint8_t>{0, 0, 0}); // zero_point
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
copy_data(input_tensors[0], x); test_case.add_expected_output<float>(
copy_data(input_tensors[1], scale); {3, 4},
copy_data(input_tensors[2], zero_point); std::vector<float>{
0.0f, 1.0f, 2.0f, 3.0f, 0.0f, 2.0f, 4.0f, 6.0f, 0.0f, 40.0f, 80.0f, 120.0f});
auto results = function->get_results(); test_case.run();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors;
result_tensors.push_back(
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape()));
auto handle = backend->compile(function);
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<float>> outputs;
outputs.push_back(read_vector<float>(result_tensors[0]));
auto expected_output = std::vector<std::vector<float>>{
{0.0f, 1.0f, 2.0f, 3.0f, 0.0f, 2.0f, 4.0f, 6.0f, 0.0f, 40.0f, 80.0f, 120.0f}};
EXPECT_TRUE(test::all_close_f(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8)
...@@ -258,39 +173,17 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8) ...@@ -258,39 +173,17 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_3.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_3.prototxt"));
auto x = std::vector<int8_t>{0, 1, 2, 3, 0, 2, 4, 6, 0, 10, 20, 30}; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
auto scale = std::vector<float>{1.0f, 2.0f, 4.0f, 8.0f};
auto zero_point = std::vector<int8_t>{0, -10, -20, -30};
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}");
auto params = function->get_parameters();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors;
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
copy_data(input_tensors[0], x); test_case.add_input(std::vector<int8_t>{0, 1, 2, 3, 0, 2, 4, 6, 0, 10, 20, 30}); // x
copy_data(input_tensors[1], scale); test_case.add_input(std::vector<float>{1.0f, 2.0f, 4.0f, 8.0f}); // scale
copy_data(input_tensors[2], zero_point); test_case.add_input(std::vector<int8_t>{0, -10, -20, -30}); // zero_point
auto results = function->get_results(); test_case.add_expected_output<float>(
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors; {3, 4},
result_tensors.push_back( std::vector<float>{
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape())); 0.0f, 22.0f, 88.0f, 264.0f, 0.0f, 24.0f, 96.0f, 288.0f, 0.0f, 40.0f, 160.0f, 480.0f});
test_case.run();
auto handle = backend->compile(function);
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<float>> outputs;
outputs.push_back(read_vector<float>(result_tensors[0]));
auto expected_output = std::vector<std::vector<float>>{
{0.0f, 22.0f, 88.0f, 264.0f, 0.0f, 24.0f, 96.0f, 288.0f, 0.0f, 40.0f, 160.0f, 480.0f}};
EXPECT_TRUE(test::all_close_f(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8_4d) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8_4d)
...@@ -298,46 +191,23 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8_4d) ...@@ -298,46 +191,23 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_int8_4d)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_4.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_4.prototxt"));
auto x = std::vector<int8_t>{7, 9, 10, 10, 5, 8, 9, 1, 8, 6, 7, 9, 10, 0, 7, 10, auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
8, 2, 6, 0, 5, 9, 8, 1, 2, 7, 5, 3, 2, 4, 1, 3,
8, 7, 4, 8, 10, 1, 5, 5, 7, 7, 0, 2, 4, 4, 0, 5};
auto scale = std::vector<float>{1.0f, 10.0f, 7.0f};
auto zero_point = std::vector<int8_t>{10, 2, 1};
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}");
auto params = function->get_parameters();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors;
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
copy_data(input_tensors[0], x); test_case.add_input(std::vector<uint8_t>{7, 9, 10, 10, 5, 8, 9, 1, 8, 6, 7, 9, 10, 0, 7, 10, 8,
copy_data(input_tensors[1], scale); 2, 6, 0, 5, 9, 8, 1, 2, 7, 5, 3, 2, 4, 1, 3, 8, 7,
copy_data(input_tensors[2], zero_point); 4, 8, 10, 1, 5, 5, 7, 7, 0, 2, 4, 4, 0, 5}); // x
test_case.add_input(std::vector<float>{1.0f, 10.0f, 7.0f}); // scale
test_case.add_input(std::vector<uint8_t>{10, 2, 1}); // zero_point
auto results = function->get_results(); test_case.add_expected_output<float>(
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors; {2, 3, 2, 4},
result_tensors.push_back( std::vector<float>{-3.0f, -1.0f, 0.0f, 0.0f, -5.0f, -2.0f, -1.0f, -9.0f, 60.0f, 40.0f,
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape())); 50.0f, 70.0f, 80.0f, -20.0f, 50.0f, 80.0f, 49.0f, 7.0f, 35.0f, -7.0f,
28.0f, 56.0f, 49.0f, 0.0f, -8.0f, -3.0f, -5.0f, -7.0f, -8.0f, -6.0f,
-9.0f, -7.0f, 60.0f, 50.0f, 20.0f, 60.0f, 80.0f, -10.0f, 30.0f, 30.0f,
42.0f, 42.0f, -7.0f, 7.0f, 21.0f, 21.0f, -7.0f, 28.0f});
auto handle = backend->compile(function); test_case.run();
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<float>> outputs;
outputs.push_back(read_vector<float>(result_tensors[0]));
auto expected_output = std::vector<std::vector<float>>{
{-3.0f, -1.0f, 0.0f, 0.0f, -5.0f, -2.0f, -1.0f, -9.0f, 60.0f, 40.0f, 50.0f, 70.0f,
80.0f, -20.0f, 50.0f, 80.0f, 49.0f, 7.0f, 35.0f, -7.0f, 28.0f, 56.0f, 49.0f, 0.0f,
-8.0f, -3.0f, -5.0f, -7.0f, -8.0f, -6.0f, -9.0f, -7.0f, 60.0f, 50.0f, 20.0f, 60.0f,
80.0f, -10.0f, 30.0f, 30.0f, 42.0f, 42.0f, -7.0f, 7.0f, 21.0f, 21.0f, -7.0f, 28.0f}};
EXPECT_TRUE(test::all_close_f(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8_negative_axis) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8_negative_axis)
...@@ -345,39 +215,17 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8_ne ...@@ -345,39 +215,17 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_dequantize_linear_1d_zero_scale_uint8_ne
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_5.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/dequantize_linear_5.prototxt"));
auto x = std::vector<uint8_t>{0, 1, 2, 3, 0, 1, 2, 3, 0, 10, 20, 30}; auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
auto scale = std::vector<float>{1.0f, 2.0f, 4.0f};
auto zero_point = std::vector<uint8_t>{0, 0, 0};
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}");
auto params = function->get_parameters();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors;
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
copy_data(input_tensors[0], x);
copy_data(input_tensors[1], scale);
copy_data(input_tensors[2], zero_point);
auto results = function->get_results(); test_case.add_input(std::vector<uint8_t>{0, 1, 2, 3, 0, 1, 2, 3, 0, 10, 20, 30}); // x
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors; test_case.add_input(std::vector<float>{1.0f, 2.0f, 4.0f}); // scale
result_tensors.push_back( test_case.add_input(std::vector<uint8_t>{0, 0, 0}); // zero_point
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape()));
auto handle = backend->compile(function); test_case.add_expected_output<float>(
handle->call_with_validate(result_tensors, input_tensors); {3, 4},
std::vector<float>{
std::vector<std::vector<float>> outputs; 0.0f, 1.0f, 2.0f, 3.0f, 0.0f, 2.0f, 4.0f, 6.0f, 0.0f, 40.0f, 80.0f, 120.0f});
outputs.push_back(read_vector<float>(result_tensors[0])); test_case.run();
auto expected_output = std::vector<std::vector<float>>{
{0.0f, 1.0f, 2.0f, 3.0f, 0.0f, 2.0f, 4.0f, 6.0f, 0.0f, 40.0f, 80.0f, 120.0f}};
EXPECT_TRUE(test::all_close_f(expected_output.front(), outputs.front()));
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear)
...@@ -408,71 +256,20 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear_2d) ...@@ -408,71 +256,20 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear_2d)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/qlinear_conv_2d.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/qlinear_conv_2d.prototxt"));
auto x = auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
read_binary_file<uint8_t>(file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/x.bin"));
auto x_scale = test_case.add_input_from_file<uint8_t>(TEST_FILES, "onnx/qlinearconv2d/x.bin");
read_binary_file<float>(file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/x_scale.bin")); test_case.add_input(std::vector<float>{0.00369204697199166f}); // x_scale
auto x_zero_point = read_binary_file<uint8_t>( test_case.add_input(std::vector<uint8_t>{132}); // x_zero_point
file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/x_zero_point.bin")); test_case.add_input(std::vector<uint8_t>{0}); // w
test_case.add_input(std::vector<float>{0.00172794575337321f}); // w_scale
auto w = test_case.add_input(std::vector<uint8_t>{255}); // w_zero_point
read_binary_file<uint8_t>(file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/w.bin")); test_case.add_input(std::vector<float>{0.00162681262008846f}); // y_scale
auto w_scale = test_case.add_input(std::vector<uint8_t>{123}); // y_zero_point
read_binary_file<float>(file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/w_scale.bin"));
auto w_zero_point = read_binary_file<uint8_t>( test_case.add_expected_output_from_file<uint8_t>(
file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/w_zero_point.bin")); {1, 1, 7, 7}, TEST_FILES, "onnx/qlinearconv2d/y.bin");
test_case.run();
auto y_scale =
read_binary_file<float>(file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/y_scale.bin"));
auto y_zero_point = read_binary_file<uint8_t>(
file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/y_zero_point.bin"));
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}");
auto params = function->get_parameters();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors;
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(3)->get_element_type(), params.at(3)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(4)->get_element_type(), params.at(4)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(5)->get_element_type(), params.at(5)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(6)->get_element_type(), params.at(6)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(7)->get_element_type(), params.at(7)->get_shape()));
copy_data(input_tensors[0], x);
copy_data(input_tensors[1], x_scale);
copy_data(input_tensors[2], x_zero_point);
copy_data(input_tensors[3], w);
copy_data(input_tensors[4], w_scale);
copy_data(input_tensors[5], w_zero_point);
copy_data(input_tensors[6], y_scale);
copy_data(input_tensors[7], y_zero_point);
auto results = function->get_results();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors;
result_tensors.push_back(
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape()));
auto handle = backend->compile(function);
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<uint8_t>> outputs;
outputs.push_back(read_vector<uint8_t>(result_tensors[0]));
std::vector<std::vector<uint8_t>> expected_output;
expected_output.push_back(
read_binary_file<uint8_t>(file_util::path_join(TEST_FILES, "onnx/qlinearconv2d/y.bin")));
EXPECT_EQ(expected_output.front(), outputs.front());
} }
NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear_3d) NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear_3d)
...@@ -480,69 +277,18 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear_3d) ...@@ -480,69 +277,18 @@ NGRAPH_TEST(onnx_${BACKEND_NAME}, model_quant_conv_linear_3d)
auto function = onnx_import::import_onnx_model( auto function = onnx_import::import_onnx_model(
file_util::path_join(SERIALIZED_ZOO, "onnx/qlinear_conv_3d.prototxt")); file_util::path_join(SERIALIZED_ZOO, "onnx/qlinear_conv_3d.prototxt"));
auto x = auto test_case = ngraph::test::NgraphTestCase(function, "${BACKEND_NAME}");
read_binary_file<uint8_t>(file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/x.bin"));
auto x_scale = test_case.add_input_from_file<uint8_t>(TEST_FILES, "onnx/qlinearconv3d/x.bin");
read_binary_file<float>(file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/x_scale.bin")); test_case.add_input(std::vector<float>{0.00389225385151803f}); // x_scale
auto x_zero_point = read_binary_file<uint8_t>( test_case.add_input(std::vector<uint8_t>{127}); // x_zero_point
file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/x_zero_point.bin")); test_case.add_input(std::vector<uint8_t>{255}); // w
test_case.add_input(std::vector<float>{0.00128723995294422f}); // w_scale
auto w = test_case.add_input(std::vector<uint8_t>{0}); // w_zero_point
read_binary_file<uint8_t>(file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/w.bin")); test_case.add_input(std::vector<float>{0.0011764180380851f}); // y_scale
auto w_scale = test_case.add_input(std::vector<uint8_t>{128}); // y_zero_point
read_binary_file<float>(file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/w_scale.bin"));
auto w_zero_point = read_binary_file<uint8_t>( test_case.add_expected_output_from_file<uint8_t>(
file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/w_zero_point.bin")); {1, 1, 4, 4, 4}, TEST_FILES, "onnx/qlinearconv3d/y.bin");
test_case.run();
auto y_scale =
read_binary_file<float>(file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/y_scale.bin"));
auto y_zero_point = read_binary_file<uint8_t>(
file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/y_zero_point.bin"));
auto backend = ngraph::runtime::Backend::create("${BACKEND_NAME}");
auto params = function->get_parameters();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> input_tensors;
input_tensors.push_back(
backend->create_tensor(params.at(0)->get_element_type(), params.at(0)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(1)->get_element_type(), params.at(1)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(2)->get_element_type(), params.at(2)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(3)->get_element_type(), params.at(3)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(4)->get_element_type(), params.at(4)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(5)->get_element_type(), params.at(5)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(6)->get_element_type(), params.at(6)->get_shape()));
input_tensors.push_back(
backend->create_tensor(params.at(7)->get_element_type(), params.at(7)->get_shape()));
copy_data(input_tensors[0], x);
copy_data(input_tensors[1], x_scale);
copy_data(input_tensors[2], x_zero_point);
copy_data(input_tensors[3], w);
copy_data(input_tensors[4], w_scale);
copy_data(input_tensors[5], w_zero_point);
copy_data(input_tensors[6], y_scale);
copy_data(input_tensors[7], y_zero_point);
auto results = function->get_results();
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> result_tensors;
result_tensors.push_back(
backend->create_tensor(results.at(0)->get_element_type(), results.at(0)->get_shape()));
auto handle = backend->compile(function);
handle->call_with_validate(result_tensors, input_tensors);
std::vector<std::vector<uint8_t>> outputs;
outputs.push_back(read_vector<uint8_t>(result_tensors[0]));
std::vector<std::vector<uint8_t>> expected_output;
expected_output.push_back(
read_binary_file<uint8_t>(file_util::path_join(TEST_FILES, "onnx/qlinearconv3d/y.bin")));
EXPECT_EQ(expected_output.front(), outputs.front());
} }
test/util/test_case.cpp
View file @ 8da348f5
...@@ -19,51 +19,48 @@ ...@@ -19,51 +19,48 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
#include "ngraph/assertion.hpp" #include "ngraph/assertion.hpp"
namespace ngraph std::map<ngraph::element::Type_t, ngraph::test::NgraphTestCase::value_comparator_function>
ngraph::test::NgraphTestCase::m_value_comparators = {
{ngraph::element::Type_t::f32, NgraphTestCase::compare_values<float>},
{ngraph::element::Type_t::f64, NgraphTestCase::compare_values<double>},
{ngraph::element::Type_t::i8, NgraphTestCase::compare_values<int8_t>},
{ngraph::element::Type_t::i16, NgraphTestCase::compare_values<int16_t>},
{ngraph::element::Type_t::i32, NgraphTestCase::compare_values<int32_t>},
{ngraph::element::Type_t::i64, NgraphTestCase::compare_values<int64_t>},
{ngraph::element::Type_t::u8, NgraphTestCase::compare_values<uint8_t>},
{ngraph::element::Type_t::u16, NgraphTestCase::compare_values<uint16_t>},
{ngraph::element::Type_t::u32, NgraphTestCase::compare_values<uint32_t>},
{ngraph::element::Type_t::u64, NgraphTestCase::compare_values<uint64_t>}};
void ngraph::test::NgraphTestCase::run()
{ {
namespace test const auto& function_results = m_function->get_results();
{ NGRAPH_CHECK(m_expected_outputs.size() == function_results.size(),
std::map<ngraph::element::Type_t, NgraphTestCase::value_comparator_function> "Expected number of outputs is different from the function's number of results.");
NgraphTestCase::value_comparators = {
{ngraph::element::Type_t::f32, NgraphTestCase::compare_values<float>},
{ngraph::element::Type_t::f64, NgraphTestCase::compare_values<double>},
{ngraph::element::Type_t::i8, NgraphTestCase::compare_values<int8_t>},
{ngraph::element::Type_t::i16, NgraphTestCase::compare_values<int16_t>},
{ngraph::element::Type_t::i32, NgraphTestCase::compare_values<int32_t>},
{ngraph::element::Type_t::i64, NgraphTestCase::compare_values<int64_t>},
{ngraph::element::Type_t::u8, NgraphTestCase::compare_values<uint8_t>},
{ngraph::element::Type_t::u16, NgraphTestCase::compare_values<uint16_t>},
{ngraph::element::Type_t::u32, NgraphTestCase::compare_values<uint32_t>},
{ngraph::element::Type_t::u64, NgraphTestCase::compare_values<uint64_t>}};
void NgraphTestCase::run() auto handle = m_backend->compile(m_function);
{ handle->call_with_validate(m_result_tensors, m_input_tensors);
const auto& function_results = m_function->get_results();
NGRAPH_CHECK(
m_expected_outputs.size() == function_results.size(),
"Expected number of outputs is different from the function's number of results.");
auto handle = m_backend->compile(m_function); for (int i = 0; i < m_expected_outputs.size(); ++i)
handle->call_with_validate(m_result_tensors, m_input_tensors); {
const auto& result_tensor = m_result_tensors.at(i);
const auto& expected_result_constant = m_expected_outputs.at(i);
const auto& element_type = result_tensor->get_element_type();
for (int i = 0; i < m_expected_outputs.size(); ++i) auto expected_shape = expected_result_constant->get_shape();
{ auto result_shape = result_tensor->get_shape();
const auto& result_tensor = m_result_tensors.at(i); EXPECT_EQ(expected_shape, result_shape);
const auto& expected_result_constant = m_expected_outputs.at(i);
const auto& element_type = result_tensor->get_element_type();
if (value_comparators.count(element_type.get_type_enum()) == 0) if (m_value_comparators.count(element_type.get_type_enum()) == 0)
{ {
NGRAPH_FAIL() << "Please add support for " << element_type NGRAPH_FAIL() << "Please add support for " << element_type
<< " to ngraph::test::NgraphTestCase::run()"; << " to ngraph::test::NgraphTestCase::run()";
} }
else else
{ {
auto values_match = value_comparators.at(element_type.get_type_enum()); auto values_match = m_value_comparators.at(element_type.get_type_enum());
EXPECT_TRUE(values_match(expected_result_constant, result_tensor)); EXPECT_TRUE(values_match(expected_result_constant, result_tensor));
}
}
} }
} }
} }
test/util/test_case.hpp
View file @ 8da348f5
...@@ -55,6 +55,20 @@ namespace ngraph ...@@ -55,6 +55,20 @@ namespace ngraph
++m_input_index; ++m_input_index;
} }
template <typename T>
void add_input_from_file(const std::string& basepath, const std::string& filename)
{
auto filepath = ngraph::file_util::path_join(basepath, filename);
add_input_from_file<T>(filepath);
}
template <typename T>
void add_input_from_file(const std::string& filepath)
{
auto value = read_binary_file<T>(filepath);
add_input(value);
}
template <typename T> template <typename T>
void add_multiple_inputs(const std::vector<std::vector<T>>& vector_of_values) void add_multiple_inputs(const std::vector<std::vector<T>>& vector_of_values)
{ {
...@@ -65,7 +79,7 @@ namespace ngraph ...@@ -65,7 +79,7 @@ namespace ngraph
} }
template <typename T> template <typename T>
void add_expected_output(const std::vector<T>& values) void add_expected_output(ngraph::Shape expected_shape, const std::vector<T>& values)
{ {
auto results = m_function->get_results(); auto results = m_function->get_results();
...@@ -78,11 +92,34 @@ namespace ngraph ...@@ -78,11 +92,34 @@ namespace ngraph
m_backend->create_tensor(function_output_type, function_output_shape)); m_backend->create_tensor(function_output_type, function_output_shape));
m_expected_outputs.emplace_back(std::make_shared<ngraph::op::Constant>( m_expected_outputs.emplace_back(std::make_shared<ngraph::op::Constant>(
function_output_type, function_output_shape, values)); function_output_type, expected_shape, values));
++m_output_index; ++m_output_index;
} }
template <typename T>
void add_expected_output(const std::vector<T>& values)
{
auto shape = m_function->get_results().at(m_output_index)->get_shape();
add_expected_output(shape, values);
}
template <typename T>
void add_expected_output_from_file(ngraph::Shape expected_shape,
const std::string& basepath,
const std::string& filename)
{
auto filepath = ngraph::file_util::path_join(basepath, filename);
add_expected_output_from_file<T>(expected_shape, filepath);
}
template <typename T>
void add_expected_output_from_file(ngraph::Shape expected_shape,
const std::string& filepath)
{
auto value = read_binary_file<T>(filepath);
add_expected_output(expected_shape, value);
}
void run(); void run();
protected: protected:
...@@ -112,8 +149,6 @@ namespace ngraph ...@@ -112,8 +149,6 @@ namespace ngraph
const std::shared_ptr<ngraph::op::Constant>&, const std::shared_ptr<ngraph::op::Constant>&,
const std::shared_ptr<ngraph::runtime::Tensor>&)>; const std::shared_ptr<ngraph::runtime::Tensor>&)>;
static std::map<ngraph::element::Type_t, value_comparator_function> value_comparators;
std::shared_ptr<Function> m_function; std::shared_ptr<Function> m_function;
std::unique_ptr<runtime::Backend> m_backend; std::unique_ptr<runtime::Backend> m_backend;
std::vector<std::shared_ptr<ngraph::runtime::Tensor>> m_input_tensors; std::vector<std::shared_ptr<ngraph::runtime::Tensor>> m_input_tensors;
...@@ -121,6 +156,7 @@ namespace ngraph ...@@ -121,6 +156,7 @@ namespace ngraph
std::vector<std::shared_ptr<ngraph::op::Constant>> m_expected_outputs; std::vector<std::shared_ptr<ngraph::op::Constant>> m_expected_outputs;
int m_input_index = 0; int m_input_index = 0;
int m_output_index = 0; int m_output_index = 0;
static std::map<ngraph::element::Type_t, value_comparator_function> m_value_comparators;
}; };
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment