Skip to content

N
ngraph
Commit c153ea8a authored by gcwenger's avatar gcwenger Committed by Robert Kimball
Browse files
Options

Graph comparison testing quiet unless problem is detected (#2258) 

* Graph comparison testing quiet unless problem is detected.

* Fixed file formatting

* Renamed ss => msg
parent 5d80f203
Show whitespace changes
Inline Side-by-side
Showing with 109 additions and 73 deletions
test/backend_graph_comparison.in.cpp
View file @ c153ea8a
......@@ -46,7 +46,8 @@ public:
void compare_results(NodeVector& result_nodes,
vector<shared_ptr<runtime::Tensor>> ref_results,
vector<shared_ptr<runtime::Tensor>> bk_results,
vector<shared_ptr<runtime::Tensor>> bk_isolated_results)
vector<shared_ptr<runtime::Tensor>> bk_isolated_results,
stringstream& msg)
{
for (int i = 0; i < ref_results.size(); ++i)
{
......@@ -55,13 +56,14 @@ public:
const shared_ptr<runtime::Tensor>& bk_isolated_data = bk_isolated_results.at(i);
std::shared_ptr<ngraph::Node> result_node = result_nodes.at(i);
cout << "Comparing results for " << result_node->get_name() << endl;
msg << "Comparing results for " << result_node->get_name() << "\n";
if (result_node->get_arguments().size() > 0)
{
cout << " inputs:" << endl;
msg << " inputs:"
<< "\n";
for (auto& p : result_node->get_arguments())
{
cout << " " << p->get_name() << " " << p->get_element_type() << endl;
msg << " " << p->get_name() << " " << p->get_element_type() << "\n";
}
}
......@@ -73,9 +75,9 @@ public:
ASSERT_EQ(ref_data->get_shape(), bk_data->get_shape());
ASSERT_EQ(ref_data->get_shape(), bk_isolated_data->get_shape());
cout << " output type: " << ref_data->get_element_type() << endl;
cout << " output shape: " << ref_data->get_shape() << endl;
cout << " output # elements: " << ref_data->get_element_count() << endl;
msg << " output type: " << ref_data->get_element_type() << "\n";
msg << " output shape: " << ref_data->get_shape() << "\n";
msg << " output # elements: " << ref_data->get_element_count() << "\n";
element::Type et = ref_data->get_element_type();
if (et == element::boolean)
......@@ -83,160 +85,182 @@ public:
vector<char> ref_data_vector = read_vector<char>(ref_data);
vector<char> bk_data_vector = read_vector<char>(bk_data);
vector<char> bk_isolated_data_vector = read_vector<char>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<char>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<char>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::f32)
{
vector<float> ref_data_vector = read_float_vector(ref_data);
vector<float> bk_data_vector = read_float_vector(bk_data);
vector<float> bk_isolated_data_vector = read_float_vector(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close_f(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close_f(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::f64)
{
vector<double> ref_data_vector = read_vector<double>(ref_data);
vector<double> bk_data_vector = read_vector<double>(bk_data);
vector<double> bk_isolated_data_vector = read_vector<double>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
// When testing with original graph dependencies test w/ loose f64 tolerance
constexpr int tolerance_bits = 30;
bool all_close_graph =
test::all_close_f(ref_data_vector, bk_data_vector, tolerance_bits);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
// When testing with isolated graph dependencies test w/ default (tight) f64 tolerance
bool all_close_isolated =
test::all_close_f(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::i8)
{
vector<int8_t> ref_data_vector = read_vector<int8_t>(ref_data);
vector<int8_t> bk_data_vector = read_vector<int8_t>(bk_data);
vector<int8_t> bk_isolated_data_vector = read_vector<int8_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<int8_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<int8_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::i16)
{
vector<int16_t> ref_data_vector = read_vector<int16_t>(ref_data);
vector<int16_t> bk_data_vector = read_vector<int16_t>(bk_data);
vector<int16_t> bk_isolated_data_vector = read_vector<int16_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<int16_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<int16_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::i32)
{
vector<int32_t> ref_data_vector = read_vector<int32_t>(ref_data);
vector<int32_t> bk_data_vector = read_vector<int32_t>(bk_data);
vector<int32_t> bk_isolated_data_vector = read_vector<int32_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<int32_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<int32_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::i64)
{
vector<int64_t> ref_data_vector = read_vector<int64_t>(ref_data);
vector<int64_t> bk_data_vector = read_vector<int64_t>(bk_data);
vector<int64_t> bk_isolated_data_vector = read_vector<int64_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<int64_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<int64_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::u8)
{
vector<uint8_t> ref_data_vector = read_vector<uint8_t>(ref_data);
vector<uint8_t> bk_data_vector = read_vector<uint8_t>(bk_data);
vector<uint8_t> bk_isolated_data_vector = read_vector<uint8_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<uint8_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<uint8_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::u16)
{
vector<uint16_t> ref_data_vector = read_vector<uint16_t>(ref_data);
vector<uint16_t> bk_data_vector = read_vector<uint16_t>(bk_data);
vector<uint16_t> bk_isolated_data_vector = read_vector<uint16_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<uint16_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<uint16_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::u32)
{
vector<uint32_t> ref_data_vector = read_vector<uint32_t>(ref_data);
vector<uint32_t> bk_data_vector = read_vector<uint32_t>(bk_data);
vector<uint32_t> bk_isolated_data_vector = read_vector<uint32_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<uint32_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<uint32_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else if (et == element::u64)
{
vector<uint64_t> ref_data_vector = read_vector<uint64_t>(ref_data);
vector<uint64_t> bk_data_vector = read_vector<uint64_t>(bk_data);
vector<uint64_t> bk_isolated_data_vector = read_vector<uint64_t>(bk_isolated_data);
cout << "Test backed op run w/ original graph dependencies:" << endl;
print_results(ref_data_vector, bk_data_vector);
msg << "Test backed op run w/ original graph dependencies:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_data_vector);
bool all_close_graph = test::all_close<uint64_t>(ref_data_vector, bk_data_vector);
cout << "Test backed op run isolated w/ inputs from ref graph run:" << endl;
print_results(ref_data_vector, bk_isolated_data_vector);
msg << "Test backed op run isolated w/ inputs from ref graph run:"
<< "\n";
msg << get_results_str(ref_data_vector, bk_isolated_data_vector);
bool all_close_isolated =
test::all_close<uint64_t>(ref_data_vector, bk_isolated_data_vector);
EXPECT_TRUE(all_close_graph && all_close_isolated);
EXPECT_TRUE(all_close_graph && all_close_isolated) << msg.str();
}
else
{
......@@ -269,7 +293,6 @@ NGRAPH_TEST_P(${BACKEND_NAME}, serialized_graph_files, compare_backends_with_gra
frozen_graph_path = file_name;
}
cout << frozen_graph_path << endl;
stringstream ss(frozen_graph_path);
shared_ptr<Function> func = ngraph::deserialize(ss);
......@@ -430,7 +453,10 @@ NGRAPH_TEST_P(${BACKEND_NAME}, serialized_graph_files, compare_backends_with_gra
handle = backend->compile(bk_isolated_func);
backend->call_with_validate(handle, bk_isolated_results, bk_args);
compare_results(new_results, ref_results, bk_results, bk_isolated_results);
// ss has path of file tested, will now be used to accumulate
// log message in case of test failure
ss << "\n";
compare_results(new_results, ref_results, bk_results, bk_isolated_results, ss);
}
// The set of graphs tested is not currently significant. These graphs were
......
test/util/test_tools.cpp
View file @ c153ea8a
......@@ -284,16 +284,20 @@ void random_init(ngraph::runtime::Tensor* tv, std::default_random_engine& engine
}
template <>
void print_results(std::vector<char>& ref_data, std::vector<char>& actual_data, size_t max_results)
string
get_results_str(std::vector<char>& ref_data, std::vector<char>& actual_data, size_t max_results)
{
stringstream ss;
size_t num_results = std::min(static_cast<size_t>(max_results), ref_data.size());
std::cout << "First " << num_results << " results";
ss << "First " << num_results << " results";
for (size_t i = 0; i < num_results; ++i)
{
std::cout << "\n"
ss << "\n"
<< std::setw(4) << i << " ref: " << std::setw(16) << std::left
<< static_cast<int>(ref_data[i]) << " actual: " << std::setw(16) << std::left
<< static_cast<int>(actual_data[i]);
}
std::cout << std::endl;
ss << "\n";
return ss.str();
}
test/util/test_tools.hpp
View file @ c153ea8a
......@@ -169,18 +169,24 @@ std::vector<std::vector<T1>> execute(const std::shared_ptr<ngraph::Function>& fu
}
template <typename T>
void print_results(std::vector<T>& ref_data, std::vector<T>& actual_data, size_t max_results = 16)
std::string
get_results_str(std::vector<T>& ref_data, std::vector<T>& actual_data, size_t max_results = 16)
{
std::stringstream ss;
size_t num_results = std::min(static_cast<size_t>(max_results), ref_data.size());
std::cout << "First " << num_results << " results";
ss << "First " << num_results << " results";
for (size_t i = 0; i < num_results; ++i)
{
std::cout << "\n"
ss << "\n"
<< std::setw(4) << i << " ref: " << std::setw(16) << std::left << ref_data[i]
<< " actual: " << std::setw(16) << std::left << actual_data[i];
}
std::cout << std::endl;
ss << "\n";
return ss.str();
}
template <>
void print_results(std::vector<char>& ref_data, std::vector<char>& actual_data, size_t max_results);
std::string get_results_str(std::vector<char>& ref_data,
std::vector<char>& actual_data,
size_t max_results);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment