Commit 40bcfdf7 authored by gcwenger's avatar gcwenger Committed by Robert Kimball

Heterogenous serialized graph testing across backends (#2020)

* Heterogenous sub-graph comparison testing

* Print index for float differences

* Disabled compare_backends_with_graphs on most backends for now. Moved to new file. Added testing of unsigned values.

* Fixed element::boolean range. Added missing include.

* Switched use of shared_ptr as parm to raw *. Moved to using namespace std in cpp. Fixed comment marker in unit_test.manifest files. Switched some EXPECT_EQ TO ASSERT_EQ. Fixed parameterized test disabling.

* Frozen naming -> serialized. Removed extraneous comments.

* Graph comparison unit test relies on CPU for reference, so only build when CPU is built.

* Reworked per backend disabling of compare_backends_with_graphs
parent 31e2765a
......@@ -138,6 +138,9 @@ set(MULTI_TEST_SRC
if(NGRAPH_DISTRIBUTED_ENABLE)
list(APPEND MULTI_TEST_SRC distributed.in.cpp)
endif()
if (NGRAPH_CPU_ENABLE)
list(APPEND MULTI_TEST_SRC backend_graph_comparison.in.cpp)
endif()
foreach(BACKEND_NAME ${ACTIVE_BACKEND_LIST})
# Some---but not all---autodiff tests go through multiple iterations with
......
//*****************************************************************************
// Copyright 2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//*****************************************************************************
#include <cstdlib>
#include <string>
#include "gtest/gtest.h"
#include "ngraph/log.hpp"
#include "ngraph/ngraph.hpp"
#include "ngraph/runtime/host_tensor.hpp"
#include "ngraph/serializer.hpp"
#include "util/all_close.hpp"
#include "util/all_close_f.hpp"
#include "util/test_control.hpp"
using namespace std;
using namespace ngraph;
static string s_manifest = "${MANIFEST}";
// clang-format off
#define TESTING_${BACKEND_NAME}_BACKEND 1
#ifdef TESTING_${BACKEND_NAME}_BACKEND // avoid macro is not used warning
#endif
// clang-format on
// Currently only used to test GPU backend, but is expected to be useful
// testing other backends (except CPU which is used as the reference backend)
#if defined(TESTING_GPU_BACKEND)
class serialized_graph_files : public ::testing::TestWithParam<string>
{
public:
void compare_results(NodeVector& result_nodes,
vector<shared_ptr<runtime::Tensor>> ref_results,
vector<shared_ptr<runtime::Tensor>> bk_results)
{
for (int i = 0; i < ref_results.size(); ++i)
{
const shared_ptr<runtime::Tensor>& ref_data = ref_results.at(i);
const shared_ptr<runtime::Tensor>& bk_data = bk_results.at(i);
cout << "Comparing results for " << result_nodes.at(i)->get_name() << endl;
if (auto node = dynamic_pointer_cast<op::GetOutputElement>(result_nodes.at(i)))
{
cout << " Parent node: ";
for (auto& p : node->get_arguments())
{
cout << " " << p->get_name() << endl;
cout << " nargs: " << p->get_arguments().size() << endl;
}
}
ASSERT_EQ(ref_data->get_element_type(), bk_data->get_element_type());
ASSERT_EQ(ref_data->get_element_count(), bk_data->get_element_count());
ASSERT_EQ(ref_data->get_shape(), bk_data->get_shape());
element::Type et = ref_data->get_element_type();
if (et == element::boolean)
{
vector<char> ref_data_vector = read_vector<char>(ref_data);
vector<char> bk_data_vector = read_vector<char>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<char>(ref_data_vector, bk_data_vector));
}
else if ((et == element::f32) || (et == element::f64))
{
vector<float> ref_data_vector = read_float_vector(ref_data);
vector<float> bk_data_vector = read_float_vector(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close_f(ref_data_vector, bk_data_vector));
}
else if (et == element::i8)
{
vector<int8_t> ref_data_vector = read_vector<int8_t>(ref_data);
vector<int8_t> bk_data_vector = read_vector<int8_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<int8_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::i16)
{
vector<int16_t> ref_data_vector = read_vector<int16_t>(ref_data);
vector<int16_t> bk_data_vector = read_vector<int16_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<int16_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::i32)
{
vector<int32_t> ref_data_vector = read_vector<int32_t>(ref_data);
vector<int32_t> bk_data_vector = read_vector<int32_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<int32_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::i64)
{
vector<int64_t> ref_data_vector = read_vector<int64_t>(ref_data);
vector<int64_t> bk_data_vector = read_vector<int64_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<int64_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::u8)
{
vector<uint8_t> ref_data_vector = read_vector<uint8_t>(ref_data);
vector<uint8_t> bk_data_vector = read_vector<uint8_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<uint8_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::u16)
{
vector<uint16_t> ref_data_vector = read_vector<uint16_t>(ref_data);
vector<uint16_t> bk_data_vector = read_vector<uint16_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<uint16_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::u32)
{
vector<uint32_t> ref_data_vector = read_vector<uint32_t>(ref_data);
vector<uint32_t> bk_data_vector = read_vector<uint32_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<uint32_t>(ref_data_vector, bk_data_vector));
}
else if (et == element::u64)
{
vector<uint64_t> ref_data_vector = read_vector<uint64_t>(ref_data);
vector<uint64_t> bk_data_vector = read_vector<uint64_t>(bk_data);
print_results(ref_data_vector, bk_data_vector);
EXPECT_TRUE(test::all_close<uint64_t>(ref_data_vector, bk_data_vector));
}
else
{
throw runtime_error("unsupported type");
}
}
}
protected:
serialized_graph_files() { file_name = GetParam(); }
string file_name;
};
NGRAPH_TEST_P(${BACKEND_NAME}, serialized_graph_files, compare_backends_with_graphs)
{
// Compare against CPU for speed. Running large graphs against the
// interpreter is too slow.
auto ref = runtime::Backend::create("CPU");
auto backend = runtime::Backend::create("${BACKEND_NAME}");
string frozen_graph_path;
if (file_name[0] != '/')
{
// Path is relative to serialized zoo path
frozen_graph_path = file_util::path_join(SERIALIZED_ZOO, file_name);
}
else
{
// Full path - useful for temporary testing to narrow down problem
frozen_graph_path = file_name;
}
cout << frozen_graph_path << endl;
stringstream ss(frozen_graph_path);
shared_ptr<Function> func = ngraph::deserialize(ss);
NodeVector new_results;
for (auto n : func->get_ordered_ops())
{
// Don't include op::Results otherwise Function c-tor will complain
if (!n->is_output() && !n->is_parameter() && !n->is_constant() &&
!(n->get_outputs().size() > 1))
{
// place conditionals here if you want to only make certain ops an output/result node
new_results.push_back(n);
}
}
//no need to include original results they are subsumed by new_results
auto new_func = make_shared<Function>(new_results, func->get_parameters());
auto ref_func = clone_function(*new_func);
auto bk_func = clone_function(*new_func);
vector<shared_ptr<runtime::Tensor>> ref_args;
vector<shared_ptr<runtime::Tensor>> bk_args;
default_random_engine engine(2112);
for (shared_ptr<op::Parameter> param : new_func->get_parameters())
{
auto data =
make_shared<ngraph::runtime::HostTensor>(param->get_element_type(), param->get_shape());
random_init(data.get(), engine);
auto ref_tensor = ref->create_tensor(param->get_element_type(), param->get_shape());
auto bk_tensor = backend->create_tensor(param->get_element_type(), param->get_shape());
ref_tensor->write(
data->get_data_ptr(), 0, data->get_element_count() * data->get_element_type().size());
bk_tensor->write(
data->get_data_ptr(), 0, data->get_element_count() * data->get_element_type().size());
ref_args.push_back(ref_tensor);
bk_args.push_back(bk_tensor);
}
vector<shared_ptr<runtime::Tensor>> ref_results;
vector<shared_ptr<runtime::Tensor>> bk_results;
ref_results.reserve(new_results.size());
bk_results.reserve(new_results.size());
for (shared_ptr<Node>& out : new_results)
{
auto ref_result = ref->create_tensor(out->get_element_type(), out->get_shape());
ref_results.push_back(ref_result);
auto bk_result = backend->create_tensor(out->get_element_type(), out->get_shape());
bk_results.push_back(bk_result);
}
if (ref_func->get_parameters().size() != ref_args.size())
{
throw ngraph::ngraph_error(
"Number of ref runtime parameters and allocated arguments don't match");
}
if (bk_func->get_parameters().size() != bk_args.size())
{
throw ngraph::ngraph_error(
"Number of backend runtime parameters and allocated arguments don't match");
}
if (ref_func->get_results().size() != ref_results.size())
{
throw ngraph::ngraph_error(
"Number of ref runtime results and allocated results don't match");
}
if (bk_func->get_results().size() != bk_results.size())
{
throw ngraph::ngraph_error(
"Number of backend runtime results and allocated results don't match");
}
ref->call_with_validate(ref_func, ref_results, ref_args);
backend->call_with_validate(bk_func, bk_results, bk_args);
compare_results(new_results, ref_results, bk_results);
}
// The set of graphs tested is not currently significant. These graphs were
// chosen because they're already availabe and demonstrate the technique.
NGRAPH_INSTANTIATE_TEST_CASE_P(
${BACKEND_NAME},
tf_resnet8_files,
serialized_graph_files,
testing::Values("tensorflow/resnet8/"
"tf_function_cluster_12[_XlaCompiledKernel=true,_XlaNumConstantArgs=3,_"
"XlaNumResourceArgs=0].v23.json",
"tensorflow/resnet8/"
"tf_function_cluster_20[_XlaCompiledKernel=true,_XlaNumConstantArgs=3,_"
"XlaNumResourceArgs=0].v23.json",
"tensorflow/resnet8/"
"tf_function_cluster_22[_XlaCompiledKernel=true,_XlaNumConstantArgs=4,_"
"XlaNumResourceArgs=0].v24.json",
"tensorflow/resnet8/"
"tf_function_cluster_23[_XlaCompiledKernel=true,_XlaNumConstantArgs=1,_"
"XlaNumResourceArgs=0].v296.json",
"tensorflow/resnet8/"
"tf_function_cluster_28[_XlaCompiledKernel=true,_XlaNumConstantArgs=0,_"
"XlaNumResourceArgs=0].v13.json",
"tensorflow/resnet8/"
"tf_function_cluster_4[_XlaCompiledKernel=true,_XlaNumConstantArgs=1,_"
"XlaNumResourceArgs=0].v14.json",
"tensorflow/resnet8/"
"tf_function_cluster_8[_XlaCompiledKernel=true,_XlaNumConstantArgs=2,_"
"XlaNumResourceArgs=0].v28.json"));
#endif // skipping tests due to backend
......@@ -34,10 +34,11 @@ namespace ngraph
/// \param atol Absolute tolerance
/// \returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
template <typename T>
bool all_close(const std::vector<T>& a,
const std::vector<T>& b,
T rtol = static_cast<T>(1e-5),
T atol = static_cast<T>(1e-8))
typename std::enable_if<std::is_signed<T>::value, bool>::type
all_close(const std::vector<T>& a,
const std::vector<T>& b,
T rtol = static_cast<T>(1e-5),
T atol = static_cast<T>(1e-8))
{
bool rc = true;
assert(a.size() == b.size());
......@@ -52,6 +53,33 @@ namespace ngraph
return rc;
}
/// \brief Same as numpy.allclose
/// \param a First tensor to compare
/// \param b Second tensor to compare
/// \param rtol Relative tolerance
/// \param atol Absolute tolerance
/// \returns true if shapes match and for all elements, |a_i-b_i| <= atol + rtol*|b_i|.
template <typename T>
typename std::enable_if<std::is_unsigned<T>::value, bool>::type
all_close(const std::vector<T>& a,
const std::vector<T>& b,
T rtol = static_cast<T>(1e-5),
T atol = static_cast<T>(1e-8))
{
bool rc = true;
assert(a.size() == b.size());
for (size_t i = 0; i < a.size(); ++i)
{
T abs_diff = (a[i] > b[i]) ? (a[i] - b[i]) : (b[i] - a[i]);
if (abs_diff > atol + rtol * b[i])
{
NGRAPH_INFO << a[i] << " is not close to " << b[i] << " at index " << i;
rc = false;
}
}
return rc;
}
/// \brief Same as numpy.allclose
/// \param a First tensor to compare
/// \param b Second tensor to compare
......
......@@ -72,7 +72,7 @@ bool test::all_close_f(const vector<float>& a,
bool is_close_f = close_f(a[i], b[i], mantissa_bits, tolerance_bits);
if (!is_close_f)
{
NGRAPH_INFO << a[i] << " is not close to " << b[i];
NGRAPH_INFO << a[i] << " is not close to " << b[i] << " at index " << i;
rc = false;
}
}
......
......@@ -120,7 +120,7 @@ namespace ngraph
::testing::internal::CodeLocation(__FILE__, __LINE__)) \
->AddTestPattern( \
#backend_name "/" #test_case_name, \
::ngraph::prepend_disabled(#test_case_name, #test_name, s_manifest).c_str(), \
::ngraph::prepend_disabled(#backend_name, #test_name, s_manifest).c_str(), \
new ::testing::internal::TestMetaFactory<NGRAPH_GTEST_TEST_CLASS_NAME_( \
backend_name, test_case_name, test_name)>()); \
return 0; \
......
......@@ -147,3 +147,119 @@ shared_ptr<Function> make_test_graph()
return f0;
}
template <>
void init_int_tv<char>(ngraph::runtime::Tensor* tv,
std::default_random_engine& engine,
char min,
char max)
{
size_t size = tv->get_element_count();
std::uniform_int_distribution<int16_t> dist(static_cast<short>(min), static_cast<short>(max));
std::vector<char> vec(size);
for (char& element : vec)
{
element = static_cast<char>(dist(engine));
}
tv->write(vec.data(), 0, vec.size() * sizeof(char));
}
template <>
void init_int_tv<int8_t>(ngraph::runtime::Tensor* tv,
std::default_random_engine& engine,
int8_t min,
int8_t max)
{
size_t size = tv->get_element_count();
std::uniform_int_distribution<int16_t> dist(static_cast<short>(min), static_cast<short>(max));
std::vector<int8_t> vec(size);
for (int8_t& element : vec)
{
element = static_cast<int8_t>(dist(engine));
}
tv->write(vec.data(), 0, vec.size() * sizeof(int8_t));
}
template <>
void init_int_tv<uint8_t>(ngraph::runtime::Tensor* tv,
std::default_random_engine& engine,
uint8_t min,
uint8_t max)
{
size_t size = tv->get_element_count();
std::uniform_int_distribution<int16_t> dist(static_cast<short>(min), static_cast<short>(max));
std::vector<uint8_t> vec(size);
for (uint8_t& element : vec)
{
element = static_cast<uint8_t>(dist(engine));
}
tv->write(vec.data(), 0, vec.size() * sizeof(uint8_t));
}
void random_init(ngraph::runtime::Tensor* tv, std::default_random_engine& engine)
{
element::Type et = tv->get_element_type();
if (et == element::boolean)
{
init_int_tv<char>(tv, engine, 0, 1);
}
else if (et == element::f32)
{
init_real_tv<float>(tv, engine, numeric_limits<float>::min(), 1.0f);
}
else if (et == element::f64)
{
init_real_tv<double>(tv, engine, numeric_limits<float>::min(), 1.0f);
}
else if (et == element::i8)
{
init_int_tv<int8_t>(tv, engine, -1, 1);
}
else if (et == element::i16)
{
init_int_tv<int16_t>(tv, engine, -1, 1);
}
else if (et == element::i32)
{
init_int_tv<int32_t>(tv, engine, 0, 1);
}
else if (et == element::i64)
{
init_int_tv<int64_t>(tv, engine, 0, 1);
}
else if (et == element::u8)
{
init_int_tv<uint8_t>(tv, engine, 0, 1);
}
else if (et == element::u16)
{
init_int_tv<uint16_t>(tv, engine, 0, 1);
}
else if (et == element::u32)
{
init_int_tv<uint32_t>(tv, engine, 0, 1);
}
else if (et == element::u64)
{
init_int_tv<uint64_t>(tv, engine, 0, 1);
}
else
{
throw runtime_error("unsupported type");
}
}
template <>
void print_results(std::vector<char>& ref_data, std::vector<char>& actual_data, size_t max_results)
{
size_t num_results = std::min(static_cast<size_t>(max_results), ref_data.size());
std::cout << "First " << num_results << " results";
for (size_t i = 0; i < num_results; ++i)
{
std::cout << "\n"
<< std::setw(4) << i << " ref: " << std::setw(16) << std::left
<< static_cast<int>(ref_data[i]) << " actual: " << std::setw(16) << std::left
<< static_cast<int>(actual_data[i]);
}
std::cout << std::endl;
}
......@@ -17,8 +17,11 @@
#pragma once
#include <exception>
#include <iomanip>
#include <iostream>
#include <list>
#include <memory>
#include <random>
#include "ngraph/descriptor/layout/tensor_layout.hpp"
#include "ngraph/file_util.hpp"
......@@ -95,6 +98,34 @@ size_t count_ops_of_type(std::shared_ptr<ngraph::Function> f)
return count;
}
template <typename T>
void init_int_tv(ngraph::runtime::Tensor* tv, std::default_random_engine& engine, T min, T max)
{
size_t size = tv->get_element_count();
std::uniform_int_distribution<T> dist(min, max);
std::vector<T> vec(size);
for (T& element : vec)
{
element = dist(engine);
}
tv->write(vec.data(), 0, vec.size() * sizeof(T));
}
template <typename T>
void init_real_tv(ngraph::runtime::Tensor* tv, std::default_random_engine& engine, T min, T max)
{
size_t size = tv->get_element_count();
std::uniform_real_distribution<T> dist(min, max);
std::vector<T> vec(size);
for (T& element : vec)
{
element = dist(engine);
}
tv->write(vec.data(), 0, vec.size() * sizeof(T));
}
void random_init(ngraph::runtime::Tensor* tv, std::default_random_engine& engine);
template <typename T, typename T1 = T>
std::vector<std::vector<T1>> execute(const std::shared_ptr<ngraph::Function>& function,
std::vector<std::vector<T>> args,
......@@ -135,3 +166,20 @@ std::vector<std::vector<T1>> execute(const std::shared_ptr<ngraph::Function>& fu
}
return result_vectors;
}
template <typename T>
void print_results(std::vector<T>& ref_data, std::vector<T>& actual_data, size_t max_results = 16)
{
size_t num_results = std::min(static_cast<size_t>(max_results), ref_data.size());
std::cout << "First " << num_results << " results";
for (size_t i = 0; i < num_results; ++i)
{
std::cout << "\n"
<< std::setw(4) << i << " ref: " << std::setw(16) << std::left << ref_data[i]
<< " actual: " << std::setw(16) << std::left << actual_data[i];
}
std::cout << std::endl;
}
template <>
void print_results(std::vector<char>& ref_data, std::vector<char>& actual_data, size_t max_results);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment