Merge pull request #15313 from andrey-golubev:map_subst_to_pattern

G-API: add transformation logic to GCompiler

* Introduce transformation logic to GCOmpiler

* Remove partialOk() method

* Fix minor issues

* Refactor code according to code review

1. Re-design matchPatternToSubstitute logic
2. Update transformations order
3. Replace check_transformations pass with a
   one time check in GCompiler ctor

* Revert unused nodes handling in pattern matching

* Address minor code review issues

* Address code review comments:

1) Fix some mistakes
2) Add new tests for endless loops
3) Update GCompiler's transformations logic

* Simplify GCompiler check for endless loops

1. Simplify transformations endless loops check:
 - Original idea wasn't a full solution
 - Need to develop a good method (heuristic?) to find loops
   in general case (TODO)
2. Remove irrelevant Endless Loops tests
3. Add new "bad arg" tests and unit tests

* Update comments

modules/gapi/CMakeLists.txt

@@ -62,7 +62,9 @@ set(gapi_srcs
     src/compiler/passes/meta.cpp
     src/compiler/passes/kernels.cpp
     src/compiler/passes/exec.cpp
+    src/compiler/passes/transformations.cpp
     src/compiler/passes/pattern_matching.cpp
+    src/compiler/passes/perform_substitution.cpp
 
     # Executor
     src/executor/gexecutor.cpp

modules/gapi/src/compiler/gcompiler.cpp

@@ -29,6 +29,7 @@
 #include "compiler/gcompiler.hpp"
 #include "compiler/gcompiled_priv.hpp"
 #include "compiler/passes/passes.hpp"
+#include "compiler/passes/pattern_matching.hpp"
 
 #include "executor/gexecutor.hpp"
 #include "backends/common/gbackend.hpp"
@@ -103,6 +104,84 @@ namespace
         return result;
     }
 
+    // Creates ADE graph from input/output proto args
+    std::unique_ptr<ade::Graph> makeGraph(const cv::GProtoArgs &ins, const cv::GProtoArgs &outs) {
+        std::unique_ptr<ade::Graph> pG(new ade::Graph);
+        ade::Graph& g = *pG;
+
+        cv::gimpl::GModel::Graph gm(g);
+        cv::gimpl::GModel::init(gm);
+        cv::gimpl::GModelBuilder builder(g);
+        auto proto_slots = builder.put(ins, outs);
+
+        // Store Computation's protocol in metadata
+        cv::gimpl::Protocol p;
+        std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots;
+        gm.metadata().set(p);
+
+        return pG;
+    }
+
+    using adeGraphs = std::vector<std::unique_ptr<ade::Graph>>;
+
+    // Creates ADE graphs (patterns and substitutes) from pkg's transformations
+    void makeTransformationGraphs(const cv::gapi::GKernelPackage& pkg,
+                                  adeGraphs& patterns,
+                                  adeGraphs& substitutes) {
+        const auto& transforms = pkg.get_transformations();
+        const auto size = transforms.size();
+        if (0u == size) return;
+
+        // pre-generate all required graphs
+        patterns.resize(size);
+        substitutes.resize(size);
+        for (auto it : ade::util::zip(ade::util::toRange(transforms),
+                                      ade::util::toRange(patterns),
+                                      ade::util::toRange(substitutes))) {
+            const auto& t = std::get<0>(it);
+            auto& p = std::get<1>(it);
+            auto& s = std::get<2>(it);
+
+            auto pattern_comp = t.pattern();
+            p = makeGraph(pattern_comp.priv().m_ins, pattern_comp.priv().m_outs);
+
+            auto substitute_comp = t.substitute();
+            s = makeGraph(substitute_comp.priv().m_ins, substitute_comp.priv().m_outs);
+        }
+    }
+
+    void checkTransformations(const cv::gapi::GKernelPackage& pkg,
+                              const adeGraphs& patterns,
+                              const adeGraphs& substitutes) {
+        const auto& transforms = pkg.get_transformations();
+        const auto size = transforms.size();
+        if (0u == size) return;
+
+        GAPI_Assert(size == patterns.size());
+        GAPI_Assert(size == substitutes.size());
+
+        const auto empty = [] (const cv::gimpl::SubgraphMatch& m) {
+            return m.inputDataNodes.empty() && m.startOpNodes.empty()
+                && m.finishOpNodes.empty() && m.outputDataNodes.empty()
+                && m.inputTestDataNodes.empty() && m.outputTestDataNodes.empty();
+        };
+
+        // **FIXME**: verify other types of endless loops. now, only checking if pattern exists in
+        //            substitute within __the same__ transformation
+        for (size_t i = 0; i < size; ++i) {
+            const auto& p = patterns[i];
+            const auto& s = substitutes[i];
+
+            auto matchInSubstitute = cv::gimpl::findMatches(*p, *s);
+            if (!empty(matchInSubstitute)) {
+                std::stringstream ss;
+                ss << "Error: (in transformation with description: '"
+                    << transforms[i].description
+                    << "') pattern is detected inside substitute";
+                throw std::runtime_error(ss.str());
+            }
+        }
+    }
 } // anonymous namespace
 
 
@@ -129,10 +208,26 @@ cv::gimpl::GCompiler::GCompiler(const cv::GComputation &c,
     // NN backends (present here via network package) always add their
     // inference kernels via auxiliary...()
 
+    // sanity check transformations
+    {
+        adeGraphs patterns, substitutes;
+        // FIXME: returning vectors of unique_ptrs from makeTransformationGraphs results in
+        //        compile error (at least) on GCC 9 with usage of copy-ctor of std::unique_ptr, so
+        //        using initialization by lvalue reference instead
+        makeTransformationGraphs(m_all_kernels, patterns, substitutes);
+        checkTransformations(m_all_kernels, patterns, substitutes);
+
+        // NB: saving generated patterns to m_all_patterns to be used later in passes
+        m_all_patterns = std::move(patterns);
+    }
+
     auto dump_path       = getGraphDumpDirectory(m_args);
 
     m_e.addPassStage("init");
     m_e.addPass("init", "check_cycles",  ade::passes::CheckCycles());
+    m_e.addPass("init", "apply_transformations",
+                std::bind(passes::applyTransformations, _1, std::cref(m_all_kernels),
+                    std::cref(m_all_patterns)));  // Note: and re-using patterns here
     m_e.addPass("init", "expand_kernels",
                 std::bind(passes::expandKernels, _1,
                           m_all_kernels)); // NB: package is copied
@@ -255,23 +350,7 @@ cv::gimpl::GCompiler::GPtr cv::gimpl::GCompiler::generateGraph()
 {
     validateInputMeta();
     validateOutProtoArgs();
-
-    // Generate ADE graph from expression-based computation
-    std::unique_ptr<ade::Graph> pG(new ade::Graph);
-    ade::Graph& g = *pG;
-
-    GModel::Graph gm(g);
-    cv::gimpl::GModel::init(gm);
-    cv::gimpl::GModelBuilder builder(g);
-    auto proto_slots = builder.put(m_c.priv().m_ins, m_c.priv().m_outs);
-    GAPI_LOG_INFO(NULL, "Generated graph: " << g.nodes().size() << " nodes" << std::endl);
-
-    // Store Computation's protocol in metadata
-    Protocol p;
-    std::tie(p.inputs, p.outputs, p.in_nhs, p.out_nhs) = proto_slots;
-    gm.metadata().set(p);
-
-    return pG;
+    return makeGraph(m_c.priv().m_ins, m_c.priv().m_outs);
 }
 
 void cv::gimpl::GCompiler::runPasses(ade::Graph &g)

modules/gapi/src/compiler/gcompiler.hpp

@@ -29,6 +29,8 @@ class GAPI_EXPORTS GCompiler
     cv::gapi::GKernelPackage m_all_kernels;
     cv::gapi::GNetPackage    m_all_networks;
 
+    std::vector<std::unique_ptr<ade::Graph>> m_all_patterns;  // built patterns from transformations
+
     void validateInputMeta();
     void validateOutProtoArgs();
 

modules/gapi/src/compiler/passes/passes.hpp

@@ -59,6 +59,10 @@ void resolveKernels(ade::passes::PassContext   &ctx,
 void fuseIslands(ade::passes::PassContext &ctx);
 void syncIslandTags(ade::passes::PassContext &ctx);
 
+void applyTransformations(ade::passes::PassContext &ctx,
+                          const gapi::GKernelPackage &pkg,
+                          const std::vector<std::unique_ptr<ade::Graph>> &preGeneratedPatterns);
+
 }} // namespace gimpl::passes
 
 } // namespace cv

modules/gapi/src/compiler/passes/pattern_matching.cpp

@@ -54,8 +54,7 @@ bool compareDataNodes(const ade::NodeHandle& first, const std::vector<std::size_
                                "shall be NodeType::DATA!");
     }
 
-    if (firstMeta.get<cv::gimpl::Data>().shape !=
-           secondMeta.get<cv::gimpl::Data>().shape) {
+    if (firstMeta.get<cv::gimpl::Data>().shape != secondMeta.get<cv::gimpl::Data>().shape) {
         return false;
     }
 
@@ -180,7 +179,7 @@ inline bool IS_ENDPOINT(const ade::NodeHandle& nh){
     // Try to rely on the nh Data::Storage::OUTPUT
     return nh->outEdges().empty();
 }
-}
+}  // anonymous namespace
 
 // Routine relies on the logic that 1 DATA node may have only 1 input edge.
 cv::gimpl::SubgraphMatch
@@ -509,7 +508,7 @@ cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph,
         // Create vector with the correctly ordered IN data nodes in the test subgraph
         std::vector<ade::NodeHandle> inputTestDataNodes;
         for (const auto& patternInNode : patternInputDataNodes) {
-            inputTestDataNodes.push_back(inputApiMatch[patternInNode]);
+            inputTestDataNodes.push_back(inputApiMatch.at(patternInNode));
         }
 
         // Traversing current result for ending OPs
@@ -560,7 +559,7 @@ cv::gimpl::findMatches(const cv::gimpl::GModel::Graph& patternGraph,
         // Create vector with the correctly ordered OUT data nodes in the test subgraph
         std::vector<ade::NodeHandle> outputTestDataNodes;
         for (const auto& patternOutNode : patternOutputDataNodes) {
-            outputTestDataNodes.push_back(outputApiMatch[patternOutNode]);
+            outputTestDataNodes.push_back(outputApiMatch.at(patternOutNode));
         }
 
         SubgraphMatch subgraph;

modules/gapi/src/compiler/passes/pattern_matching.hpp

@@ -41,7 +41,6 @@ namespace gimpl {
             return    !inputDataNodes.empty() && !startOpNodes.empty()
                    && !finishOpNodes.empty() && !outputDataNodes.empty()
                    && !inputTestDataNodes.empty() && !outputTestDataNodes.empty();
-
         }
 
        S nodes() const {
@@ -93,6 +92,11 @@ namespace gimpl {
     GAPI_EXPORTS SubgraphMatch findMatches(const cv::gimpl::GModel::Graph& patternGraph,
                                            const cv::gimpl::GModel::Graph& compGraph);
 
+    GAPI_EXPORTS void performSubstitution(cv::gimpl::GModel::Graph& graph,
+                                          const cv::gimpl::Protocol& patternP,
+                                          const cv::gimpl::Protocol& substituteP,
+                                          const cv::gimpl::SubgraphMatch& patternToGraphMatch);
+
 } //namespace gimpl
 } //namespace cv
 #endif // OPENCV_GAPI_PATTERN_MATCHING_HPP

modules/gapi/src/compiler/passes/perform_substitution.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2019 Intel Corporation
+
+#include "pattern_matching.hpp"
+
+#include "ade/util/zip_range.hpp"
+
+namespace cv { namespace gimpl {
+namespace {
+using Graph = GModel::Graph;
+
+template<typename Iterator>
+ade::NodeHandle getNh(Iterator it) { return *it; }
+
+template<>
+ade::NodeHandle getNh(SubgraphMatch::M::const_iterator it) { return it->second; }
+
+template<typename Container>
+void erase(Graph& g, const Container& c)
+{
+    for (auto first = c.begin(); first != c.end(); ++first) {
+        ade::NodeHandle node = getNh(first);
+        if (node == nullptr) continue;  // some nodes might already be erased
+        g.erase(node);
+    }
+}
+}  // anonymous namespace
+
+void performSubstitution(GModel::Graph& graph,
+                         const Protocol& patternP,
+                         const Protocol& substituteP,
+                         const SubgraphMatch& patternToGraphMatch)
+{
+    // 1. substitute input nodes
+    const auto& patternIns = patternP.in_nhs;
+    const auto& substituteIns = substituteP.in_nhs;
+
+    for (auto it : ade::util::zip(ade::util::toRange(patternIns),
+                                  ade::util::toRange(substituteIns))) {
+        // Note: we don't replace input DATA nodes here, only redirect their output edges
+        const auto& patternDataNode = std::get<0>(it);
+        const auto& substituteDataNode = std::get<1>(it);
+        const auto& graphDataNode = patternToGraphMatch.inputDataNodes.at(patternDataNode);
+        GModel::redirectReaders(graph, substituteDataNode, graphDataNode);
+    }
+
+    // 2. substitute output nodes
+    const auto& patternOuts = patternP.out_nhs;
+    const auto& substituteOuts = substituteP.out_nhs;
+
+    for (auto it : ade::util::zip(ade::util::toRange(patternOuts),
+                                  ade::util::toRange(substituteOuts))) {
+        // Note: we don't replace output DATA nodes here, only redirect their input edges
+        const auto& patternDataNode = std::get<0>(it);
+        const auto& substituteDataNode = std::get<1>(it);
+        const auto& graphDataNode = patternToGraphMatch.outputDataNodes.at(patternDataNode);
+        // delete existing edges (otherwise we cannot redirect)
+        for (auto e : graphDataNode->inEdges()) {
+            graph.erase(e);
+        }
+        GModel::redirectWriter(graph, substituteDataNode, graphDataNode);
+    }
+
+    // 3. erase redundant nodes:
+    // erase input data nodes of __substitute__
+    erase(graph, substituteIns);
+
+    // erase old start OP nodes of __main graph__
+    erase(graph, patternToGraphMatch.startOpNodes);
+
+    // erase old internal nodes of __main graph__
+    erase(graph, patternToGraphMatch.internalLayers);
+
+    // erase old finish OP nodes of __main graph__
+    erase(graph, patternToGraphMatch.finishOpNodes);
+
+    // erase output data nodes of __substitute__
+    erase(graph, substituteOuts);
+}
+
+}  // namespace gimpl
+}  // namespace cv

modules/gapi/src/compiler/passes/transformations.cpp

+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+//
+// Copyright (C) 2019 Intel Corporation
+
+#include "precomp.hpp"
+
+#include <ade/util/zip_range.hpp>
+#include <ade/graph.hpp>
+
+#include "api/gcomputation_priv.hpp"
+
+#include "compiler/gmodel.hpp"
+#include "compiler/gmodelbuilder.hpp"
+#include "compiler/passes/passes.hpp"
+#include "compiler/passes/pattern_matching.hpp"
+
+#include <sstream>
+
+namespace cv { namespace gimpl { namespace passes {
+namespace
+{
+using Graph = GModel::Graph;
+using Metadata = typename Graph::CMetadataT;
+
+// Checks pairs of {pattern node, substitute node} and asserts if there are any incompatibilities
+void checkDataNodes(const Graph& pattern,
+                    const Graph& substitute,
+                    const std::vector<ade::NodeHandle>& patternNodes,
+                    const std::vector<ade::NodeHandle>& substituteNodes)
+{
+    for (auto it : ade::util::zip(patternNodes, substituteNodes)) {
+        auto pNodeMeta = pattern.metadata(std::get<0>(it));
+        auto sNodeMeta = substitute.metadata(std::get<1>(it));
+        GAPI_Assert(pNodeMeta.get<NodeType>().t == NodeType::DATA);
+        GAPI_Assert(pNodeMeta.get<NodeType>().t == sNodeMeta.get<NodeType>().t);
+        GAPI_Assert(pNodeMeta.get<Data>().shape == sNodeMeta.get<Data>().shape);
+    }
+}
+
+// Checks compatibility of pattern and substitute graphs based on in/out nodes
+void checkCompatibility(const Graph& pattern,
+                        const Graph& substitute,
+                        const Protocol& patternP,
+                        const Protocol& substituteP)
+{
+    const auto& patternDataInputs = patternP.in_nhs;
+    const auto& patternDataOutputs = patternP.out_nhs;
+
+    const auto& substituteDataInputs = substituteP.in_nhs;
+    const auto& substituteDataOutputs = substituteP.out_nhs;
+
+    // number of data nodes must be the same
+    GAPI_Assert(patternDataInputs.size() == substituteDataInputs.size());
+    GAPI_Assert(patternDataOutputs.size() == substituteDataOutputs.size());
+
+    // for each pattern input node, verify a corresponding substitute input node
+    checkDataNodes(pattern, substitute, patternDataInputs, substituteDataInputs);
+
+    // for each pattern output node, verify a corresponding substitute output node
+    checkDataNodes(pattern, substitute, patternDataOutputs, substituteDataOutputs);
+}
+
+// Tries to substitute __single__ pattern with substitute in the given graph
+bool tryToSubstitute(ade::Graph& main,
+                     const std::unique_ptr<ade::Graph>& patternG,
+                     const cv::GComputation& substitute)
+{
+    GModel::Graph gm(main);
+
+    // 1. find a pattern in main graph
+    auto match1 = findMatches(*patternG, gm);
+    if (!match1.ok()) {
+        return false;
+    }
+
+    // 2. build substitute graph inside the main graph
+    cv::gimpl::GModelBuilder builder(main);
+    const auto& proto_slots = builder.put(substitute.priv().m_ins, substitute.priv().m_outs);
+    Protocol substituteP;
+    std::tie(substituteP.inputs, substituteP.outputs, substituteP.in_nhs, substituteP.out_nhs) =
+        proto_slots;
+
+    const Protocol& patternP = GModel::Graph(*patternG).metadata().get<Protocol>();
+
+    // 3. check that pattern and substitute are compatible
+    // FIXME: in theory, we should always have compatible pattern/substitute. if not, we're in
+    //        half-completed state where some transformations are already applied - what can we do
+    //        to handle the situation better?  -- use transactional API as in fuse_islands pass?
+    checkCompatibility(*patternG, gm, patternP, substituteP);
+
+    // 4. make substitution
+    performSubstitution(gm, patternP, substituteP, match1);
+
+    return true;
+}
+}  // anonymous namespace
+
+void applyTransformations(ade::passes::PassContext& ctx,
+                          const gapi::GKernelPackage& pkg,
+                          const std::vector<std::unique_ptr<ade::Graph>>& patterns)
+{
+    const auto& transforms = pkg.get_transformations();
+    const auto size = transforms.size();
+    if (0u == size) return;
+    // Note: patterns are already generated at this point
+    GAPI_Assert(patterns.size() == transforms.size());
+
+    // transform as long as it is possible
+    bool canTransform = true;
+    while (canTransform)
+    {
+        canTransform = false;
+
+        // iterate through every transformation and try to transform graph parts
+        for (auto it : ade::util::zip(ade::util::toRange(transforms), ade::util::toRange(patterns)))
+        {
+            const auto& t = std::get<0>(it);
+            auto& pattern = std::get<1>(it);  // Note: using pre-created graphs
+            GAPI_Assert(nullptr != pattern);
+
+            // if transformation is successful (pattern found and substituted), it is possible that
+            // other transformations will also be successful, so set canTransform to the returned
+            // value from tryToSubstitute
+            canTransform = tryToSubstitute(ctx.graph, pattern, t.substitute());
+
+            // Note: apply the __same__ substitution as many times as possible and only after go to
+            //       the next one. BUT it can happen that after applying some substitution, some
+            //       _previous_ patterns will also be found and these will be applied first
+            if (canTransform) {
+                break;
+            }
+        }
+    }
+}
+}  // namespace passes
+}  // namespace gimpl
+}  // namespace cv