Skip to content

N
ngraph
Commit b3b5b9fd authored by gcwenger's avatar gcwenger Committed by Scott Cyphers
Browse files
Options

Support for parameterized testing combined with backend driven manifests (#1855) 

* Support for parameterized testing combined with backend driven manifests.

* clang-format fix

* Improved nGraph variants of gtest fixture and parameterized testing macros

* Reverted mistaken change to src/ngraph/runtime/gpu/CMakeLists.txt
parent 02affea5
Show whitespace changes
Inline Side-by-side
Showing with 190 additions and 58 deletions
src/ngraph/runtime/intelgpu/unit_test.manifest
View file @ b3b5b9fd
avg_pool_2d_2channel_2image_padded_only_above_do_not_include_in_computation
avg_pool_2d_2channel_2image_padded_only_above_include_in_computation
avg_pool_3d_strided_uneven_padded_do_not_include_in_computation
avg_pool_3d_uneven_strided_padded_include_in_computation
avg_pool_3d_uneven_strided_padded
backwards_batch_norm_three_outputs
backwards_dot_scalar_tensor
backwards_dot_tensor3_tensor3
......
test/backend_pool.in.cpp
View file @ b3b5b9fd
......@@ -237,11 +237,20 @@ NGRAPH_TEST(${BACKEND_NAME}, max_pool_2d_2channel_2image_asym_pad)
read_vector<float>(result));
}
NGRAPH_TEST(${BACKEND_NAME}, max_pool_2d_1channel_1image_overpadded)
// MaxPool2D1ChannelTests test fixture for test setup reuse
class MaxPool2D1ChannelTests : public testing::Test
{
public:
Shape shape_a{1, 1, 5, 5};
Shape window_shape{2, 3};
auto window_movement_strides = Strides{1, 1};
Strides window_movement_strides{1, 1};
protected:
virtual void SetUp() override {}
};
NGRAPH_TEST_F(${BACKEND_NAME}, MaxPool2D1ChannelTests, max_pool_2d_1channel_1image_overpadded)
{
Shape padding_below{2, 0};
Shape padding_above{1, 2};
auto A = make_shared<op::Parameter>(element::f32, shape_a);
......@@ -277,11 +286,8 @@ NGRAPH_TEST(${BACKEND_NAME}, max_pool_2d_1channel_1image_overpadded)
read_vector<float>(result)));
}
NGRAPH_TEST(${BACKEND_NAME}, max_pool_2d_1channel_1image_padded)
NGRAPH_TEST_F(${BACKEND_NAME}, MaxPool2D1ChannelTests, max_pool_2d_1channel_1image_padded)
{
Shape shape_a{1, 1, 5, 5};
Shape window_shape{2, 3};
auto window_movement_strides = Strides{1, 1};
Shape padding_below{1, 0};
Shape padding_above{1, 2};
auto A = make_shared<op::Parameter>(element::f32, shape_a);
......@@ -1183,7 +1189,16 @@ NGRAPH_TEST(${BACKEND_NAME},
read_vector<float>(result)));
}
NGRAPH_TEST(${BACKEND_NAME}, avg_pool_3d_strided_uneven_padded_do_not_include_in_computation)
// Params to drive avg_pool_3d testing variations
class avg_pool_3d_params : public ::testing::TestWithParam<bool>
{
protected:
avg_pool_3d_params() { include_pad = GetParam(); }
bool include_pad;
};
// avg_pool_3d test code using params
NGRAPH_TEST_P(${BACKEND_NAME}, avg_pool_3d_params, avg_pool_3d_uneven_strided_padded)
{
Shape shape_a{64, 3, 12, 13, 15};
Shape window_shape{4, 5, 4};
......@@ -1195,11 +1210,11 @@ NGRAPH_TEST(${BACKEND_NAME}, avg_pool_3d_strided_uneven_padded_do_not_include_in
auto cpu_f = make_shared<Function>(
make_shared<op::AvgPool>(
A, window_shape, move_strides, padding_below, padding_above, false),
A, window_shape, move_strides, padding_below, padding_above, include_pad),
op::ParameterVector{A});
auto int_f = make_shared<Function>(
make_shared<op::AvgPool>(
B, window_shape, move_strides, padding_below, padding_above, false),
B, window_shape, move_strides, padding_below, padding_above, include_pad),
op::ParameterVector{B});
test::Uniform<float> rng(0.0f, 1.0f);
vector<vector<float>> args;
......@@ -1218,35 +1233,5 @@ NGRAPH_TEST(${BACKEND_NAME}, avg_pool_3d_strided_uneven_padded_do_not_include_in
}
}
NGRAPH_TEST(${BACKEND_NAME}, avg_pool_3d_uneven_strided_padded_include_in_computation)
{
Shape shape_a{64, 3, 7, 8, 10};
Shape window_shape{2, 3, 2};
auto move_strides = Strides{2, 3, 4};
Shape padding_below{5, 6, 4};
Shape padding_above{6, 4, 5};
auto A = make_shared<op::Parameter>(element::f32, shape_a);
auto B = make_shared<op::Parameter>(element::f32, shape_a);
auto cpu_f = make_shared<Function>(
make_shared<op::AvgPool>(A, window_shape, move_strides, padding_below, padding_above, true),
op::ParameterVector{A});
auto int_f = make_shared<Function>(
make_shared<op::AvgPool>(B, window_shape, move_strides, padding_below, padding_above, true),
op::ParameterVector{B});
test::Uniform<float> rng(0.0f, 1.0f);
vector<vector<float>> args;
for (shared_ptr<op::Parameter> param : int_f->get_parameters())
{
vector<float> tensor_val(shape_size(param->get_shape()));
rng.initialize(tensor_val);
args.push_back(tensor_val);
}
auto int_results = execute(int_f, args, "INTERPRETER");
auto backend_results = execute(cpu_f, args, "${BACKEND_NAME}");
for (size_t i = 0; i < backend_results.size(); i++)
{
EXPECT_TRUE(test::all_close(backend_results.at(i), int_results.at(i), 1.0e-4f, 1.0e-4f));
}
}
// avg_pool_3d case generation
NGRAPH_INSTANTIATE_TEST_CASE_P(${BACKEND_NAME}, include_pad, avg_pool_3d_params, testing::Bool());
test/util/test_control.cpp
View file @ b3b5b9fd
......@@ -27,12 +27,12 @@ using namespace ngraph;
static unordered_map<string, unordered_set<string>> s_blacklists;
string ngraph::prepend_disabled(const string& test_case_name,
string ngraph::prepend_disabled(const string& backend_name,
const string& test_name,
const string& manifest)
{
string rc = test_name;
unordered_set<string>& blacklist = s_blacklists[test_case_name];
unordered_set<string>& blacklist = s_blacklists[backend_name];
if (blacklist.empty() && !manifest.empty())
{
ifstream f(manifest);
......@@ -54,3 +54,16 @@ string ngraph::prepend_disabled(const string& test_case_name,
}
return rc;
}
string ngraph::combine_test_backend_and_case(const string& backend_name,
const string& test_casename)
{
if (backend_name == test_casename)
{
return backend_name;
}
else
{
return backend_name + "_" + test_casename;
}
}
test/util/test_control.hpp
View file @ b3b5b9fd
......@@ -22,37 +22,172 @@
namespace ngraph
{
std::string prepend_disabled(const std::string& test_case_name,
std::string prepend_disabled(const std::string& backend_name,
const std::string& test_name,
const std::string& manifest);
std::string combine_test_backend_and_case(const std::string& backend_name,
const std::string& test_casename);
}
#define NGRAPH_GTEST_TEST_(test_case_name, test_name, parent_class, parent_id) \
class GTEST_TEST_CLASS_NAME_(test_case_name, test_name) \
#define NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name) \
backend_name##_##test_case_name##_##test_name##_Test
#define NGRAPH_GTEST_TEST_(backend_name, test_case_name, test_name, parent_class, parent_id) \
class NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name) \
: public parent_class \
{ \
public: \
GTEST_TEST_CLASS_NAME_(test_case_name, test_name)() {} \
NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name)() {} \
private: \
virtual void TestBody(); \
static ::testing::TestInfo* const test_info_ GTEST_ATTRIBUTE_UNUSED_; \
GTEST_DISALLOW_COPY_AND_ASSIGN_(GTEST_TEST_CLASS_NAME_(test_case_name, test_name)); \
GTEST_DISALLOW_COPY_AND_ASSIGN_(NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, \
test_case_name, \
test_name)); \
}; \
\
::testing::TestInfo* const GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::test_info_ = \
::testing::TestInfo* const NGRAPH_GTEST_TEST_CLASS_NAME_( \
backend_name, test_case_name, test_name)::test_info_ = \
::testing::internal::MakeAndRegisterTestInfo( \
#test_case_name, \
::ngraph::prepend_disabled(#test_case_name, #test_name, s_manifest).c_str(), \
::ngraph::combine_test_backend_and_case(#backend_name, #test_case_name).c_str(), \
::ngraph::prepend_disabled(#backend_name, #test_name, s_manifest).c_str(), \
nullptr, \
nullptr, \
::testing::internal::CodeLocation(__FILE__, __LINE__), \
(parent_id), \
parent_class::SetUpTestCase, \
parent_class::TearDownTestCase, \
new ::testing::internal::TestFactoryImpl<GTEST_TEST_CLASS_NAME_(test_case_name, \
test_name)>); \
void GTEST_TEST_CLASS_NAME_(test_case_name, test_name)::TestBody()
new ::testing::internal::TestFactoryImpl<NGRAPH_GTEST_TEST_CLASS_NAME_( \
backend_name, test_case_name, test_name)>); \
void NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name)::TestBody()
#define NGRAPH_TEST(test_case_name, test_name) \
NGRAPH_GTEST_TEST_( \
test_case_name, test_name, ::testing::Test, ::testing::internal::GetTestTypeId())
NGRAPH_GTEST_TEST_(test_case_name, \
test_case_name, \
test_name, \
::testing::Test, \
::testing::internal::GetTestTypeId())
// NGRAPH_TEST_F facilitates the use of the same configuration parameters for multiple
// unit tests similar to the original TEST_F, but with the introduction of a new 0th
// parameter for the backend name, which allows nGraph's manifest controlled unit testing.
//
// Start by defining a class derived from testing::Test, which you'll pass for the
// text_fixture parameter.
// Then use this class to define multiple related unit tests (which share some common
// configuration information and/or setup code).
//
// Generated test names take the form:
// BACKENDNAME/FixtureClassName.test_name
// where the test case name is "BACKENDNAME/FixtureClassName"
// and the test name is "test_name"
//
// With the use of NGRAPH_TEST_F the filter to run all the tests for a given backend
// should be:
// --gtest_filter=BACKENDNAME*.*
// (rather than the BACKENDNAME.* that worked before the use of NGRAPH_TEST_F)
#define NGRAPH_TEST_F(backend_name, test_fixture, test_name) \
NGRAPH_GTEST_TEST_(backend_name, \
test_fixture, \
test_name, \
test_fixture, \
::testing::internal::GetTypeId<test_fixture>())
// NGRAPH_TEST_P combined with NGRAPH_INSTANTIATE_TEST_CASE_P facilate the generation
// of value parameterized tests (similar to the original TEST_P and INSTANTIATE_TEST_CASE_P
// with the addition of a new 0th parameter for the backend name, which allows nGraph's
// manifest controlled unit testing).
//
// Start by defining a class derived from ::testing::TestWithParam<T>, which you'll pass
// for the test_case_name parameter.
// Then use NGRAPH_INSTANTIATE_TEST_CASE_P to define each generation of test cases (see below).
#define NGRAPH_TEST_P(backend_name, test_case_name, test_name) \
class NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name) \
: public test_case_name \
{ \
public: \
NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name)() {} \
virtual void TestBody(); \
\
private: \
static int AddToRegistry() \
{ \
::testing::UnitTest::GetInstance() \
->parameterized_test_registry() \
.GetTestCasePatternHolder<test_case_name>( \
#backend_name "/" #test_case_name, \
::testing::internal::CodeLocation(__FILE__, __LINE__)) \
->AddTestPattern( \
#backend_name "/" #test_case_name, \
::ngraph::prepend_disabled(#test_case_name, #test_name, s_manifest).c_str(), \
new ::testing::internal::TestMetaFactory<NGRAPH_GTEST_TEST_CLASS_NAME_( \
backend_name, test_case_name, test_name)>()); \
return 0; \
} \
static int gtest_registering_dummy_ GTEST_ATTRIBUTE_UNUSED_; \
GTEST_DISALLOW_COPY_AND_ASSIGN_(NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, \
test_case_name, \
test_name)); \
}; \
int NGRAPH_GTEST_TEST_CLASS_NAME_( \
backend_name, test_case_name, test_name)::gtest_registering_dummy_ = \
NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name)::AddToRegistry(); \
void NGRAPH_GTEST_TEST_CLASS_NAME_(backend_name, test_case_name, test_name)::TestBody()
// Use NGRAPH_INSTANTIATE_TEST_CASE_P to create a generated set of test case variations.
// The prefix parameter is a label that you optionally provide (no quotes) for a unique
// test name (helpful for labelling a set of inputs and for filtering).
// The prefix parameter can be skipped by simply using a bare comma (see example below).
//
// Unlike INSTANTIATE_TEST_CASE_P we don't currently support passing a custom param
// name generator. Supporting this with a single macro name requires the use of
// ... and __VA_ARGS__ which in turn generates a warning using INSTANTIATE_TEST_CASE_P
// without a trailing , parameter.
//
// Examples:
// NGRAPH_INSTANTIATE_TEST_CASE_P(BACKENDNAME, // backend_name
// , // empty/skipped prefix
// TestWithParamSubClass, // test_case_name
// ::testing::Range(0, 3) ) // test generator
// would generate:
// BACKENDNAME/TestWithParamSubClass.test_name/0
// BACKENDNAME/TestWithParamSubClass.test_name/1
// BACKENDNAME/TestWithParamSubClass.test_name/2
//
// NGRAPH_INSTANTIATE_TEST_CASE_P(BACKENDNAME, // backend_name
// NumericRangeTests, // prefix
// TestWithParamSubClass, // test_case_name
// ::testing::Range(0, 3) ) // test generator
// would generate:
// BACKENDNAME/NumericRangeTests/BACKENDNAME/TestWithParamSubClass.test_name/0
// BACKENDNAME/NumericRangeTests/BACKENDNAME/TestWithParamSubClass.test_name/1
// BACKENDNAME/NumericRangeTests/BACKENDNAME/TestWithParamSubClass.test_name/2
//
// With the use of NGRAPH_TEST_P and NGRAPH_INSTANTIATE_TEST_CASE_P
// the filter to run all the tests for a given backend should be:
// --gtest_filter=BACKENDNAME*.*
// (rather than the BACKENDNAME.* that worked before the use of NGRAPH_TEST_P)
#define NGRAPH_INSTANTIATE_TEST_CASE_P(backend_name, prefix, test_case_name, generator) \
::testing::internal::ParamGenerator<test_case_name::ParamType> \
gtest_##prefix##backend_name##test_case_name##_EvalGenerator_() \
{ \
return generator; \
} \
::std::string gtest_##prefix##backend_name##test_case_name##_EvalGenerateName_( \
const ::testing::TestParamInfo<test_case_name::ParamType>& info) \
{ \
return ::testing::internal::GetParamNameGen<test_case_name::ParamType>()(info); \
} \
static int gtest_##prefix##backend_name##test_case_name##_dummy_ GTEST_ATTRIBUTE_UNUSED_ = \
::testing::UnitTest::GetInstance() \
->parameterized_test_registry() \
.GetTestCasePatternHolder<test_case_name>( \
#backend_name "/" #test_case_name, \
::testing::internal::CodeLocation(__FILE__, __LINE__)) \
->AddTestCaseInstantiation( \
#prefix[0] != '\0' ? #backend_name "/" #prefix : "", \
&gtest_##prefix##backend_name##test_case_name##_EvalGenerator_, \
&gtest_##prefix##backend_name##test_case_name##_EvalGenerateName_, \
__FILE__, \
__LINE__)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment