Skip to content

O
opencv
Commit 1b517a45 authored by Li Peng's avatar Li Peng
Browse files
Options

add fp16 accuracy and perf test 

Signed-off-by: 's avatarLi Peng <peng.li@intel.com>
parent ba5e8bef
Hide whitespace changes
Inline Side-by-side
Showing with 74 additions and 39 deletions
modules/dnn/perf/perf_net.cpp
View file @ 1b517a45
......@@ -121,7 +121,9 @@ PERF_TEST_P_(DNNTestNetwork, Inception_5h)
PERF_TEST_P_(DNNTestNetwork, ENet)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
if ((backend == DNN_BACKEND_INFERENCE_ENGINE) ||
(backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("");
processNet("dnn/Enet-model-best.net", "", "enet.yml",
Mat(cv::Size(512, 256), CV_32FC3));
}
......@@ -232,7 +234,8 @@ const tuple<DNNBackend, DNNTarget> testCases[] = {
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL_FP16),
#endif
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_CPU),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL)
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL_FP16)
};
INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, testing::ValuesIn(testCases));
......
modules/dnn/test/test_backends.cpp
View file @ 1b517a45
......@@ -147,7 +147,9 @@ TEST_P(DNNTestNetwork, Inception_5h)
TEST_P(DNNTestNetwork, ENet)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE) throw SkipTestException("");
if ((backend == DNN_BACKEND_INFERENCE_ENGINE) ||
(backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("");
processNet("dnn/Enet-model-best.net", "", Size(512, 512), "l367_Deconvolution",
target == DNN_TARGET_OPENCL ? "dnn/halide_scheduler_opencl_enet.yml" :
"dnn/halide_scheduler_enet.yml",
......@@ -161,9 +163,11 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_Caffe)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);
float l1 = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.0007 : 0.0;
float lInf = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.011 : 0.0;
processNet("dnn/MobileNetSSD_deploy.caffemodel", "dnn/MobileNetSSD_deploy.prototxt",
inp, "detection_out");
inp, "detection_out", "", l1, lInf);
}
TEST_P(DNNTestNetwork, MobileNet_SSD_TensorFlow)
......@@ -173,15 +177,17 @@ TEST_P(DNNTestNetwork, MobileNet_SSD_TensorFlow)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);
float l1 = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.008 : 0.0;
float lInf = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.06 : 0.0;
processNet("dnn/ssd_mobilenet_v1_coco.pb", "dnn/ssd_mobilenet_v1_coco.pbtxt",
inp, "detection_out");
inp, "detection_out", "", l1, lInf);
}
TEST_P(DNNTestNetwork, SSD_VGG16)
{
if (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL ||
backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU ||
backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU)
if ((backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ||
(backend == DNN_BACKEND_HALIDE && target == DNN_TARGET_CPU) ||
(backend == DNN_BACKEND_INFERENCE_ENGINE && target != DNN_TARGET_CPU))
throw SkipTestException("");
processNet("dnn/VGG_ILSVRC2016_SSD_300x300_iter_440000.caffemodel",
"dnn/ssd_vgg16.prototxt", Size(300, 300), "detection_out");
......@@ -236,14 +242,17 @@ TEST_P(DNNTestNetwork, Inception_v2_SSD_TensorFlow)
throw SkipTestException("");
Mat sample = imread(findDataFile("dnn/street.png", false));
Mat inp = blobFromImage(sample, 1.0f / 127.5, Size(300, 300), Scalar(127.5, 127.5, 127.5), false);
float l1 = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.008 : 0.0;
float lInf = (backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ? 0.07 : 0.0;
processNet("dnn/ssd_inception_v2_coco_2017_11_17.pb", "dnn/ssd_inception_v2_coco_2017_11_17.pbtxt",
inp, "detection_out");
inp, "detection_out", "", l1, lInf);
}
TEST_P(DNNTestNetwork, DenseNet_121)
{
if (backend == DNN_BACKEND_HALIDE ||
backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16)
if ((backend == DNN_BACKEND_HALIDE) ||
(backend == DNN_BACKEND_DEFAULT && target == DNN_TARGET_OPENCL_FP16) ||
(backend == DNN_BACKEND_INFERENCE_ENGINE && target == DNN_TARGET_OPENCL_FP16))
throw SkipTestException("");
processNet("dnn/DenseNet_121.caffemodel", "dnn/DenseNet_121.prototxt", Size(224, 224), "", "caffe");
}
......@@ -258,7 +267,8 @@ const tuple<DNNBackend, DNNTarget> testCases[] = {
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_INFERENCE_ENGINE, DNN_TARGET_OPENCL_FP16),
#endif
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL)
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL),
tuple<DNNBackend, DNNTarget>(DNN_BACKEND_DEFAULT, DNN_TARGET_OPENCL_FP16)
};
INSTANTIATE_TEST_CASE_P(/*nothing*/, DNNTestNetwork, testing::ValuesIn(testCases));
......
modules/dnn/test/test_caffe_importer.cpp
View file @ 1b517a45
......@@ -104,7 +104,11 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
ASSERT_FALSE(net.empty());
}
net.setPreferableTarget(get<1>(GetParam()));
int targetId = get<1>(GetParam());
const float l1 = 1e-5;
const float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 3e-3 : 1e-4;
net.setPreferableTarget(targetId);
Mat sample = imread(_tf("grace_hopper_227.png"));
ASSERT_TRUE(!sample.empty());
......@@ -112,10 +116,11 @@ TEST_P(Reproducibility_AlexNet, Accuracy)
net.setInput(blobFromImage(sample, 1.0f, Size(227, 227), Scalar(), false), "data");
Mat out = net.forward("prob");
Mat ref = blobFromNPY(_tf("caffe_alexnet_prob.npy"));
normAssert(ref, out);
normAssert(ref, out, "", l1, lInf);
}
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(), availableDnnTargets()));
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_AlexNet, Combine(testing::Bool(),
Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16)));
#if !defined(_WIN32) || defined(_WIN64)
TEST(Reproducibility_FCN, Accuracy)
......@@ -176,8 +181,11 @@ TEST_P(Reproducibility_MobileNet_SSD, Accuracy)
const string proto = findDataFile("dnn/MobileNetSSD_deploy.prototxt", false);
const string model = findDataFile("dnn/MobileNetSSD_deploy.caffemodel", false);
Net net = readNetFromCaffe(proto, model);
int targetId = GetParam();
const float l1 = (targetId == DNN_TARGET_OPENCL_FP16) ? 1.5e-4 : 1e-5;
const float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 4e-4 : 1e-4;
net.setPreferableTarget(GetParam());
net.setPreferableTarget(targetId);
Mat sample = imread(_tf("street.png"));
......@@ -185,8 +193,10 @@ TEST_P(Reproducibility_MobileNet_SSD, Accuracy)
net.setInput(inp);
Mat out = net.forward();
const float scores_diff = (targetId == DNN_TARGET_OPENCL_FP16) ? 4e-4 : 1e-5;
const float boxes_iou_diff = (targetId == DNN_TARGET_OPENCL_FP16) ? 5e-3 : 1e-4;
Mat ref = blobFromNPY(_tf("mobilenet_ssd_caffe_out.npy"));
normAssertDetections(ref, out);
normAssertDetections(ref, out, "", 0.0, scores_diff, boxes_iou_diff);
// Check that detections aren't preserved.
inp.setTo(0.0f);
......@@ -212,10 +222,12 @@ TEST_P(Reproducibility_MobileNet_SSD, Accuracy)
// a single sample in batch. The first numbers of detection vectors are batch id.
outBatch = outBatch.reshape(1, outBatch.total() / 7);
EXPECT_EQ(outBatch.rows, 2 * numDetections);
normAssert(outBatch.rowRange(0, numDetections), ref);
normAssert(outBatch.rowRange(numDetections, 2 * numDetections).colRange(1, 7), ref.colRange(1, 7));
normAssert(outBatch.rowRange(0, numDetections), ref, "", l1, lInf);
normAssert(outBatch.rowRange(numDetections, 2 * numDetections).colRange(1, 7), ref.colRange(1, 7),
"", l1, lInf);
}
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD, availableDnnTargets());
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_MobileNet_SSD,
Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
typedef testing::TestWithParam<DNNTarget> Reproducibility_ResNet50;
TEST_P(Reproducibility_ResNet50, Accuracy)
......@@ -226,6 +238,9 @@ TEST_P(Reproducibility_ResNet50, Accuracy)
int targetId = GetParam();
net.setPreferableTarget(targetId);
float l1 = (targetId == DNN_TARGET_OPENCL_FP16) ? 3e-5 : 1e-5;
float lInf = (targetId == DNN_TARGET_OPENCL_FP16) ? 6e-3 : 1e-4;
Mat input = blobFromImage(imread(_tf("googlenet_0.png")), 1.0f, Size(224,224), Scalar(), false);
ASSERT_TRUE(!input.empty());
......@@ -233,20 +248,21 @@ TEST_P(Reproducibility_ResNet50, Accuracy)
Mat out = net.forward();
Mat ref = blobFromNPY(_tf("resnet50_prob.npy"));
normAssert(ref, out);
normAssert(ref, out, "", l1, lInf);
if (targetId == DNN_TARGET_OPENCL)
if (targetId == DNN_TARGET_OPENCL || targetId == DNN_TARGET_OPENCL_FP16)
{
UMat out_umat;
net.forward(out_umat);
normAssert(ref, out_umat, "out_umat");
normAssert(ref, out_umat, "out_umat", l1, lInf);
std::vector<UMat> out_umats;
net.forward(out_umats);
normAssert(ref, out_umats[0], "out_umat_vector");
normAssert(ref, out_umats[0], "out_umat_vector", l1, lInf);
}
}
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50, availableDnnTargets());
INSTANTIATE_TEST_CASE_P(/**/, Reproducibility_ResNet50,
Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
typedef testing::TestWithParam<DNNTarget> Reproducibility_SqueezeNet_v1_1;
TEST_P(Reproducibility_SqueezeNet_v1_1, Accuracy)
......
modules/dnn/test/test_tf_importer.cpp
View file @ 1b517a45
......@@ -295,24 +295,30 @@ TEST_P(Test_TensorFlow_nets, opencv_face_detector_uint8)
INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_nets, availableDnnTargets());
TEST(Test_TensorFlow, defun)
typedef testing::TestWithParam<DNNTarget> Test_TensorFlow_fp16;
TEST_P(Test_TensorFlow_fp16, tests)
{
runTensorFlowNet("defun_dropout");
int targetId = GetParam();
const float l1 = 7e-4;
const float lInf = 1e-2;
runTensorFlowNet("fp16_single_conv", targetId, false, l1, lInf);
runTensorFlowNet("fp16_deconvolution", targetId, false, l1, lInf);
runTensorFlowNet("fp16_max_pool_odd_same", targetId, false, l1, lInf);
runTensorFlowNet("fp16_padding_valid", targetId, false, l1, lInf);
runTensorFlowNet("fp16_eltwise_add_mul", targetId, false, l1, lInf);
runTensorFlowNet("fp16_max_pool_odd_valid", targetId, false, l1, lInf);
runTensorFlowNet("fp16_pad_and_concat", targetId, false, l1, lInf);
runTensorFlowNet("fp16_max_pool_even", targetId, false, l1, lInf);
runTensorFlowNet("fp16_padding_same", targetId, false, l1, lInf);
}
TEST(Test_TensorFlow, fp16)
INSTANTIATE_TEST_CASE_P(/**/, Test_TensorFlow_fp16,
Values(DNN_TARGET_CPU, DNN_TARGET_OPENCL, DNN_TARGET_OPENCL_FP16));
TEST(Test_TensorFlow, defun)
{
const float l1 = 1e-3;
const float lInf = 1e-2;
runTensorFlowNet("fp16_single_conv", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_deconvolution", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_max_pool_odd_same", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_padding_valid", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_eltwise_add_mul", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_max_pool_odd_valid", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_pad_and_concat", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_max_pool_even", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("fp16_padding_same", DNN_TARGET_CPU, false, l1, lInf);
runTensorFlowNet("defun_dropout");
}
TEST(Test_TensorFlow, quantized)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment