Skip to content

O
opencv
Commit 9e332dc5 authored by Dmitry Kurtaev's avatar Dmitry Kurtaev
Browse files
Options

Broadcasting from ONNX

parent a694e507
Hide whitespace changes
Inline Side-by-side
Showing with 111 additions and 62 deletions
modules/dnn/src/layers/scale_layer.cpp
View file @ 9e332dc5
......@@ -46,14 +46,14 @@ public:
{
std::vector<Mat> inputs;
inputs_arr.getMatVector(inputs);
hasWeights = blobs.size() == 2 || (blobs.size() == 1 && !hasBias);
hasWeights = blobs.size() == 2 || (blobs.size() <= 1 && !hasBias);
CV_Assert((inputs.size() == 2 && blobs.empty()) || blobs.size() == (int)hasWeights + (int)hasBias);
}
virtual bool supportBackend(int backendId) CV_OVERRIDE
{
return backendId == DNN_BACKEND_OPENCV || backendId == DNN_BACKEND_HALIDE ||
(backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1) ||
(backendId == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019 && axis == 1 && !blobs.empty()) ||
(backendId == DNN_BACKEND_INFERENCE_ENGINE_NGRAPH && axis > 0);
}
......@@ -78,10 +78,9 @@ public:
Mat &outBlob = outputs[0];
// There is a mode when we multiply a first blob by a second one
// instead of trainable weights.
Mat weights = blobs.empty() ? inputs[1] : (hasWeights ? blobs[0] : Mat());
Mat bias = hasBias ? blobs.back().reshape(1, 1) : Mat();
if (!weights.empty())
weights = weights.reshape(1, 1);
Mat weights = hasWeights ? (blobs.empty() ? inputs[1] : blobs[0]).reshape(1, 1) : Mat();;
Mat bias = hasBias ? (blobs.empty() ? inputs[1] : blobs.back()).reshape(1, 1) : Mat();
MatShape inpShape = shape(inpBlob);
const int numWeights = !weights.empty() ? weights.total() : bias.total();
CV_Assert(numWeights != 0);
......@@ -229,28 +228,40 @@ public:
#ifdef HAVE_DNN_NGRAPH
virtual Ptr<BackendNode> initNgraph(const std::vector<Ptr<BackendWrapper> >& inputs, const std::vector<Ptr<BackendNode> >& nodes) CV_OVERRIDE
{
CV_Assert(!blobs.empty());
const size_t numChannels = blobs[0].total();
auto ieInpNode = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
auto ieInpNode0 = nodes[0].dynamicCast<InfEngineNgraphNode>()->node;
auto ieInpNode1 = nodes.size() > 1 ? nodes[1].dynamicCast<InfEngineNgraphNode>()->node : nullptr;
size_t numChannels = 1;
if (blobs.empty())
for (const size_t& dim : ieInpNode1->get_shape())
numChannels *= dim;
else
numChannels = blobs[0].total();
std::vector<size_t> shape(ieInpNode->get_shape().size(), 1);
std::vector<size_t> shape(ieInpNode0->get_shape().size(), 1);
int cAxis = clamp(axis, shape.size());
shape[cAxis] = numChannels;
auto node = ieInpNode;
auto node = ieInpNode0;
if (hasWeights)
{
auto weight = std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
ngraph::Shape(shape), blobs[0].data);
auto weight = blobs.empty() ? ieInpNode1 :
std::make_shared<ngraph::op::Constant>(ngraph::element::f32, ngraph::Shape(shape), blobs[0].data);
node = std::make_shared<ngraph::op::v1::Multiply>(node, weight, ngraph::op::AutoBroadcastType::NUMPY);
}
if (hasBias || !hasWeights)
{
auto bias = hasBias ?
std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
ngraph::Shape(shape), blobs.back().data) :
std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
ngraph::Shape(shape), std::vector<float>(numChannels, 0).data());
std::shared_ptr<ngraph::Node> bias;
if (hasBias)
{
bias = blobs.empty() ? ieInpNode1 :
std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
ngraph::Shape(shape), blobs.back().data);
}
else
bias = std::make_shared<ngraph::op::Constant>(ngraph::element::f32,
ngraph::Shape(shape), std::vector<float>(numChannels, 0).data());
node = std::make_shared<ngraph::op::v1::Add>(node, bias, ngraph::op::AutoBroadcastType::NUMPY);
}
return Ptr<BackendNode>(new InfEngineNgraphNode(node));
......@@ -259,8 +270,8 @@ public:
void getScaleShift(Mat& scale, Mat& shift) const CV_OVERRIDE
{
scale = hasWeights ? blobs[0] : Mat();
shift = hasBias ? blobs.back() : Mat();
scale = (hasWeights && !blobs.empty()) ? blobs[0] : Mat();
shift = (hasBias && !blobs.empty()) ? blobs.back() : Mat();
}
virtual int64 getFLOPS(const std::vector<MatShape> &inputs,
......
modules/dnn/src/onnx/onnx_importer.cpp
View file @ 9e332dc5
......@@ -427,24 +427,57 @@ void ONNXImporter::populateNet(Net dstNet)
}
layerParams.type = "Slice";
}
else if (layer_type == "Add" || layer_type == "Sum")
else if (layer_type == "Add" || layer_type == "Sum" || layer_type == "Sub")
{
bool isSub = layer_type == "Sub";
CV_CheckEQ(node_proto.input_size(), 2, "");
if (layer_id.find(node_proto.input(1)) == layer_id.end())
{
Mat blob = getBlob(node_proto, constBlobs, 1);
blob = blob.reshape(1, 1);
if (blob.total() == 1) {
layerParams.type = "Power";
layerParams.set("shift", blob.at<float>(0));
layerParams.set("shift", (isSub ? -1 : 1) * blob.at<float>(0));
}
else {
layerParams.type = "Scale";
layerParams.set("bias_term", true);
layerParams.blobs.push_back(blob);
layerParams.blobs.push_back((isSub ? -1 : 1) * blob);
}
}
else {
else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
{
layerParams.type = "Eltwise";
if (isSub)
{
static float subCoeffs[] = {1.f, -1.f};
layerParams.set("coeff", DictValue::arrayReal<float*>(subCoeffs, 2));
}
}
else
{
if (isSub)
{
LayerParams powerParams;
powerParams.name = layerParams.name + "/neg";
powerParams.type = "Power";
powerParams.set("scale", -1);
//Create Power layer
int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
//Connect to input
layerId = layer_id.find(node_proto.input(1));
CV_Assert(layerId != layer_id.end());
dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
//Add shape
layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
outShapes[powerParams.name] = outShapes[node_proto.input(1)];
//Replace input to Power
node_proto.set_input(1, powerParams.name);
}
layerParams.type = "Scale";
layerParams.set("bias_term", true);
}
}
else if (layer_type == "Max")
......@@ -452,19 +485,6 @@ void ONNXImporter::populateNet(Net dstNet)
layerParams.type = "Eltwise";
layerParams.set("operation", "max");
}
else if (layer_type == "Sub")
{
Mat blob = getBlob(node_proto, constBlobs, 1);
if (blob.total() == 1) {
layerParams.type = "Power";
layerParams.set("shift", -blob.at<float>(0));
}
else {
layerParams.type = "Scale";
layerParams.set("has_bias", true);
layerParams.blobs.push_back(-1.0f * blob.reshape(1, 1));
}
}
else if (layer_type == "Neg")
{
layerParams.type = "Power";
......@@ -643,10 +663,35 @@ void ONNXImporter::populateNet(Net dstNet)
layerParams.type = "Scale";
}
}
else {
else if (outShapes[node_proto.input(0)] == outShapes[node_proto.input(1)])
{
layerParams.type = "Eltwise";
layerParams.set("operation", isDiv ? "div" : "prod");
}
else
{
if (isDiv)
{
LayerParams powerParams;
powerParams.name = layerParams.name + "/inv";
powerParams.type = "Power";
powerParams.set("power", -1);
//Create Power layer
int id = dstNet.addLayer(powerParams.name, powerParams.type, powerParams);
//Connect to input
layerId = layer_id.find(node_proto.input(1));
CV_Assert(layerId != layer_id.end());
dstNet.connect(layerId->second.layerId, layerId->second.outputId, id, 0);
//Add shape
layer_id.insert(std::make_pair(powerParams.name, LayerInfo(id, 0)));
outShapes[powerParams.name] = outShapes[node_proto.input(1)];
//Replace input to Power
node_proto.set_input(1, powerParams.name);
}
layerParams.type = "Scale";
}
if (!haveVariables)
{
......
modules/dnn/test/test_onnx_importer.cpp
View file @ 9e332dc5
......@@ -32,29 +32,33 @@ public:
void testONNXModels(const String& basename, const Extension ext = npy,
const double l1 = 0, const float lInf = 0, const bool useSoftmax = false,
bool checkNoFallbacks = true)
bool checkNoFallbacks = true, int numInps = 1)
{
String onnxmodel = _tf("models/" + basename + ".onnx", required);
Mat inp, ref;
std::vector<Mat> inps(numInps);
Mat ref;
if (ext == npy) {
inp = blobFromNPY(_tf("data/input_" + basename + ".npy"));
for (int i = 0; i < numInps; ++i)
inps[i] = blobFromNPY(_tf("data/input_" + basename + (numInps > 1 ? format("_%d", i) : "") + ".npy"));
ref = blobFromNPY(_tf("data/output_" + basename + ".npy"));
}
else if (ext == pb) {
inp = readTensorFromONNX(_tf("data/input_" + basename + ".pb"));
for (int i = 0; i < numInps; ++i)
inps[i] = readTensorFromONNX(_tf("data/input_" + basename + (numInps > 1 ? format("_%d", i) : "") + ".pb"));
ref = readTensorFromONNX(_tf("data/output_" + basename + ".pb"));
}
else
CV_Error(Error::StsUnsupportedFormat, "Unsupported extension");
checkBackend(&inp, &ref);
checkBackend(&inps[0], &ref);
Net net = readNetFromONNX(onnxmodel);
ASSERT_FALSE(net.empty());
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
net.setInput(inp);
for (int i = 0; i < numInps; ++i)
net.setInput(inps[i], numInps > 1 ? format("%d", i) : "");
Mat out = net.forward("");
if (useSoftmax)
......@@ -328,25 +332,14 @@ TEST_P(Test_ONNX_layers, ResizeUnfused)
TEST_P(Test_ONNX_layers, MultyInputs)
{
const String model = _tf("models/multy_inputs.onnx");
Net net = readNetFromONNX(model);
ASSERT_FALSE(net.empty());
net.setPreferableBackend(backend);
net.setPreferableTarget(target);
Mat inp1 = blobFromNPY(_tf("data/input_multy_inputs_0.npy"));
Mat inp2 = blobFromNPY(_tf("data/input_multy_inputs_1.npy"));
Mat ref = blobFromNPY(_tf("data/output_multy_inputs.npy"));
checkBackend(&inp1, &ref);
net.setInput(inp1, "0");
net.setInput(inp2, "1");
Mat out = net.forward();
testONNXModels("multy_inputs", npy, 0, 0, false, true, 2);
}
normAssert(ref, out, "", default_l1, default_lInf);
expectNoFallbacksFromIE(net);
TEST_P(Test_ONNX_layers, Broadcast)
{
if (backend == DNN_BACKEND_INFERENCE_ENGINE_NN_BUILDER_2019)
applyTestTag(CV_TEST_TAG_DNN_SKIP_IE_NN_BUILDER);
testONNXModels("channel_broadcast", npy, 0, 0, false, true, 2);
}
TEST_P(Test_ONNX_layers, Div)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment