op.py 19.4 KB
Newer Older
openvino-pushbot's avatar
openvino-pushbot committed
1
"""
2
 Copyright (c) 2018-2019 Intel Corporation
openvino-pushbot's avatar
openvino-pushbot committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

17
import copy
openvino-pushbot's avatar
openvino-pushbot committed
18
import logging as log
Alexey Suhov's avatar
Alexey Suhov committed
19
from collections import namedtuple
openvino-pushbot's avatar
openvino-pushbot committed
20 21 22 23 24 25

import networkx as nx
import numpy as np

from mo.front.extractor import add_attrs_props
from mo.front.extractor import update_ie_fields
26
from mo.graph.graph import Node, Graph
openvino-pushbot's avatar
openvino-pushbot committed
27 28 29 30 31 32 33 34 35 36
from mo.utils import class_registration
from mo.utils.error import Error


class Op(object):
    registered_ops = {}
    registered_cls = []
    # Add the derived class to excluded_classes if one should not be registered in registered_ops
    excluded_classes = []

37
    def __init__(self, graph: Graph, attrs1: dict = None, attrs2: dict = None):
openvino-pushbot's avatar
openvino-pushbot committed
38
        self.graph = graph
Alexey Suhov's avatar
Alexey Suhov committed
39 40 41 42 43
        try:
            self.ir_version = graph.graph['ir_version']
        except:
            self.ir_version = None

openvino-pushbot's avatar
openvino-pushbot committed
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59
        self.attrs = {
            'precision': "FP32",
            'kind': 'op'
        }
        self.default_backend_attrs = []
        if attrs1 is not None:
            self.attrs.update(attrs1)
        if attrs2 is not None:
            self.attrs.update(attrs2)

    def add_node(self, attrs: dict = None):
        new_attrs = {}
        new_attrs.update(self.attrs)
        if attrs is not None:
            new_attrs.update(attrs)
        id_prefix = new_attrs['name'] if 'name' in new_attrs else ''
60
        id = self.graph.unique_id(id_prefix)
openvino-pushbot's avatar
openvino-pushbot committed
61 62
        new_attrs['name'] = id
        new_attrs = add_attrs_props(new_attrs)
Alexey Suhov's avatar
Alexey Suhov committed
63
        update_ie_fields(new_attrs, self.ir_version)
openvino-pushbot's avatar
openvino-pushbot committed
64 65
        self.substitute_ie_attrs(new_attrs)
        self.graph.add_node(id, **new_attrs)
66 67 68

        node = Node(self.graph, id)
        return node
openvino-pushbot's avatar
openvino-pushbot committed
69 70 71 72 73 74

    def substitute_ie_attrs(self, new_attrs: dict):
        """
        Replace standard list of attribute in layer/data by attributes
        delivered by backend_attrs
        """
Alexey Suhov's avatar
Alexey Suhov committed
75 76
        backend_attrs_mapping = {
            None: self.backend_attrs,
77 78
            10: self.backend_attrs,
            6: self.backend_attrs,
79
            5: self.backend_attrs,
80
            4: self.backend_attrs,
Alexey Suhov's avatar
Alexey Suhov committed
81 82 83 84 85 86
            3: self.backend_attrs,
            2: self.backend_attrs_v2
        }

        if self.ir_version not in backend_attrs_mapping.keys():
            raise Error("Unrecognized IR version was specified: {}".format(self.ir_version))
openvino-pushbot's avatar
openvino-pushbot committed
87 88 89 90 91 92

        new_attrs.update({
            'IE': [(
                'layer',
                [('id', lambda node: node.node), 'name', 'precision', 'type'],
                [
Alexey Suhov's avatar
Alexey Suhov committed
93
                    ('data', backend_attrs_mapping[self.ir_version]() + self.default_backend_attrs, []),
openvino-pushbot's avatar
openvino-pushbot committed
94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
                    '@ports',
                    '@consts'])]
        })

    @staticmethod
    def extract_port(node_port):
        if isinstance(node_port, tuple):
            node = node_port[0]
            port = node_port[1]
        else:
            node = node_port
            port = 0
        # 'data' nodes do not have 'out' edge attibute but always has one output
        out_ids = [attr['out'] for _, __, attr in node.graph.out_edges(node.id, data=True) if 'out' in attr]
        if len(set(out_ids)) > 1 and not isinstance(node_port, tuple):
            raise Error('Node {} has more than one outputs. Provide output port explicitly. '.format(node.name))
        return node, port

112
    def create_node_on_port(self, node: Node, out_port: int, attrs: dict = None, edge_attrs: dict = None):
openvino-pushbot's avatar
openvino-pushbot committed
113 114 115 116 117 118
        """
        Removes an edge, that is connected to nodes out_port. Creates new_node with attrs attributes and
        connects it to node by edge that stores the same information as cutted edge.
        :param node: Input node, to cut the edge from
        :param out_port: output port of edge to cut
        :param attrs: attributes of new node
119
        :param edge_attrs: attributes to be changed/added to new edge
openvino-pushbot's avatar
openvino-pushbot committed
120 121
        :return: Node instance of created new_node
        """
122 123 124 125 126
        if edge_attrs is None:
            edge_attrs = {'in': 0}
        prev_edge_attrs = copy.deepcopy(node.out_edge(out_port))
        prev_edge_attrs.update(edge_attrs)
        new_edge_attrs = prev_edge_attrs
openvino-pushbot's avatar
openvino-pushbot committed
127 128 129
        if attrs is None:
            attrs = dict()
        new_node = self.add_node(attrs)
130
        self.graph.add_edge(node.id, new_node.id, **new_edge_attrs)
openvino-pushbot's avatar
openvino-pushbot committed
131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
        return new_node

    def create_node(self, inputs: list = None, attrs: dict = None, edge_attrs: dict = None):
        # TODO pass also edge attributes to copy to newly created edges
        # TODO attrs should be matched with attrs()
        if inputs is not None:
            inputs = [Op.extract_port(inp) for inp in inputs]
        else:
            inputs = []
        if attrs is None:
            attrs = dict()
        new_node = self.add_node(attrs)
        # Missed careful handling of debug information
        for i, inp in enumerate(inputs):
            edge_attr = {'in': i, 'out': inp[1],
Alexey Suhov's avatar
Alexey Suhov committed
146 147
                         'in_attrs': ['in', 'permutation'],
                         'out_attrs': ['out', 'permutation'],
openvino-pushbot's avatar
openvino-pushbot committed
148
                         'data_attrs': []} if not inp[0].has_valid('kind') or inp[0].kind == 'op' \
Alexey Suhov's avatar
Alexey Suhov committed
149
                else {'in': i, 'in_attrs': ['in', 'permutation']}
openvino-pushbot's avatar
openvino-pushbot committed
150 151
            if edge_attrs is not None:
                edge_attr.update(edge_attrs)
152 153
            new_node.add_input_port(i, skip_if_exist=True)
            new_node.add_output_port(inp[1], skip_if_exist=True)
openvino-pushbot's avatar
openvino-pushbot committed
154 155 156 157
            self.graph.add_edge(inp[0].id, new_node.id, **edge_attr)
        return new_node

    def create_node_with_data(self, inputs: list = None, attrs: dict = None,
Alexey Suhov's avatar
Alexey Suhov committed
158
                              data_nodes: [Node, np.ndarray, list] = None, edge_attrs: list = None):
openvino-pushbot's avatar
openvino-pushbot committed
159 160 161 162
        """
        Creates a new node with given inputs and attrs and also creates data node that
        holds the op output value. Inputs should be data nodes (not op nodes).
        Work for ops with a single output port only.
Alexey Suhov's avatar
Alexey Suhov committed
163
        Edge attributes in edge_attrs go in order of items in 'inputs'
openvino-pushbot's avatar
openvino-pushbot committed
164 165 166 167 168 169 170 171
        """
        if inputs is None:
            inputs = []
        if attrs is None:
            attrs = {}
        # No need to extract port, because input node should be a data node,
        # so there is no choice.
        new_op_node = self.add_node(attrs)
Alexey Suhov's avatar
Alexey Suhov committed
172

openvino-pushbot's avatar
openvino-pushbot committed
173
        # TODO Preserve debug infor
Alexey Suhov's avatar
Alexey Suhov committed
174 175
        inputs_with_edge_attrs = []
        for i, inp in enumerate(inputs):
176 177
            if inp is None:
                continue
Alexey Suhov's avatar
Alexey Suhov committed
178 179 180 181 182 183 184
            edge_attr = {'in': i}
            if edge_attrs is not None and i < len(edge_attrs):
                edge_attr.update(edge_attrs[i])
            inputs_with_edge_attrs.append((inp.id, new_op_node.id, edge_attr))
        
        self.graph.add_edges_from(inputs_with_edge_attrs)
        
openvino-pushbot's avatar
openvino-pushbot committed
185 186 187 188
        # TODO: Extend to the case when multiple output ports
        old_data_value = [None]
        old_data_shape = [None]
        if data_nodes is None:
189
            data_node = self.graph.unique_id()
openvino-pushbot's avatar
openvino-pushbot committed
190 191 192 193 194 195 196 197 198 199 200 201 202
            self.graph.add_node(data_node, **add_attrs_props(
                dict(kind='data', precision="FP32", name=data_node, value=None, shape=None, data_type=None,
                     infer=None)))
            data_nodes = [Node(self.graph, data_node)]
        else:
            if type(data_nodes) not in [list, np.ndarray]:
                data_nodes = [data_nodes]
            old_data_value = [data_node.value.copy() if data_node.has_valid('value') else None for data_node in
                              data_nodes]
            old_data_shape = [data_node.shape.copy() if data_node.has_valid('shape') else None for data_node in
                              data_nodes]
        for id, data_node in enumerate(data_nodes):
            self.graph.add_edges_from([(new_op_node.id, data_node.id, {'out': id})])
203

openvino-pushbot's avatar
openvino-pushbot committed
204
        if new_op_node.has_valid('infer'):
205 206 207
            if log.getLogger().isEnabledFor(log.DEBUG):
                log.debug('Start running infer function for individual op node with attributes: {}'
                          ''.format(str(new_op_node)))
openvino-pushbot's avatar
openvino-pushbot committed
208
            new_op_node.infer(new_op_node)
209 210 211
            if new_op_node.has('nchw_layout'):
                for out_node in new_op_node.out_nodes().values():
                    out_node['nchw_layout'] = new_op_node.nchw_layout
openvino-pushbot's avatar
openvino-pushbot committed
212 213 214
            assert all(old_value is None for old_value in old_data_value) or all(
                [np.array_equal(old_data_value[id], data_node.value) for id, data_node in enumerate(data_nodes)])
            assert all(old_shape is None for old_shape in old_data_shape) or all(
215 216 217 218 219
                [np.array_equal(old_data_shape[id], data_node.shape) for id, data_node in enumerate(data_nodes)]), \
                "After re-inference of {} node, old and new shapes do not match. Old shapes: {}, new shapes: {}.".format(
                    new_op_node.soft_get('name'),
                    [old_data_shape[id] for id in range(len(data_nodes))],
                    [data_node.shape for data_node in data_nodes])
openvino-pushbot's avatar
openvino-pushbot committed
220
            for data_node in data_nodes:
221 222 223
                if log.getLogger().isEnabledFor(log.DEBUG):
                    log.debug(
                        'Finished running infer function, data nodes attributes: {}'.format(data_node))
openvino-pushbot's avatar
openvino-pushbot committed
224 225 226
        return data_nodes[0] if len(data_nodes) == 1 else data_nodes

    @staticmethod
227
    def create_data_node(graph: Graph, op_node: Node, attrs: dict = None, edge_attrs: dict = None, out_port=0):
openvino-pushbot's avatar
openvino-pushbot committed
228
        assert op_node is not None and op_node.kind == 'op'
229 230
        assert out_port not in op_node.out_nodes()

openvino-pushbot's avatar
openvino-pushbot committed
231 232 233
        if attrs is None:
            attrs = {}

234
        data_node = graph.unique_id(op_node.id)
openvino-pushbot's avatar
openvino-pushbot committed
235 236 237 238 239
        defaul_attrs = dict(kind='data', precision="FP32", name=data_node, value=None, shape=None, data_type=None,
                            infer=None)
        defaul_attrs.update(attrs)
        graph.add_node(data_node, **add_attrs_props(defaul_attrs))
        data_node = Node(graph, data_node)
Alexey Suhov's avatar
Alexey Suhov committed
240
        if edge_attrs is not None:
241
            graph.add_edges_from([(op_node.id, data_node.id, {'out': out_port, **edge_attrs})])
Alexey Suhov's avatar
Alexey Suhov committed
242
        else:
243
            graph.add_edges_from([(op_node.id, data_node.id, {'out': out_port})])
openvino-pushbot's avatar
openvino-pushbot committed
244 245 246
        return data_node

    @staticmethod
247
    def _create_data_node(graph: Graph, name: str, attrs: dict = None):
openvino-pushbot's avatar
openvino-pushbot committed
248 249 250
        if attrs is None:
            attrs = {}

251
        data_node = graph.unique_id(name)
openvino-pushbot's avatar
openvino-pushbot committed
252 253 254 255 256 257 258 259
        defaul_attrs = dict(kind='data', precision="FP32", name=data_node, value=None, shape=None, data_type=None,
                            infer=None)
        defaul_attrs.update(attrs)
        graph.add_node(data_node, **add_attrs_props(defaul_attrs))
        data_node = Node(graph, data_node)
        return data_node

    @staticmethod
260 261 262 263
    def create_input_data_node(graph: Graph, name: str, value: np.array, attrs: dict = {}):
        data_node = graph.unique_id(name)
        defaul_attrs = dict(kind='data', precision="FP32", name=data_node, value=np.array(value),
                            shape=np.array(value.shape),
openvino-pushbot's avatar
openvino-pushbot committed
264 265 266 267 268
                            data_type=None, infer=None)
        defaul_attrs.update(attrs)
        graph.add_node(data_node, **add_attrs_props(defaul_attrs))
        return Node(graph, data_node)

Alexey Suhov's avatar
Alexey Suhov committed
269
    @staticmethod
270
    def create_and_connect_input_data_node(graph: Graph, op_node: Node, attrs: dict = None, edge_attrs: dict = None):
Alexey Suhov's avatar
Alexey Suhov committed
271 272 273 274 275 276
        assert op_node is not None and op_node.kind == 'op'
        if attrs is None:
            attrs = {}
        if edge_attrs is None:
            edge_attrs = {}

277
        data_node = graph.unique_id(op_node.id)
Alexey Suhov's avatar
Alexey Suhov committed
278 279 280 281 282
        defaul_attrs = dict(kind='data', precision="FP32", name=data_node, value=None, shape=None, data_type=None,
                            infer=None)
        defaul_attrs.update(attrs)
        graph.add_node(data_node, **add_attrs_props(defaul_attrs))
        data_node = Node(graph, data_node)
283
        op_node.add_input_port(edge_attrs['in'], skip_if_exist=True)
Alexey Suhov's avatar
Alexey Suhov committed
284 285 286
        graph.add_edges_from([(data_node.id, op_node.id, edge_attrs)])
        return data_node

openvino-pushbot's avatar
openvino-pushbot committed
287 288 289 290 291 292 293 294 295
    def update_node(self, node: Node, attrs: dict = None):
        """
        Updates/creates new attributes in node based on self.attrs and attrs.
        """
        new_attrs = {}
        new_attrs.update(self.attrs)
        if attrs:
            new_attrs.update(attrs)
        new_attrs = add_attrs_props(new_attrs)
Alexey Suhov's avatar
Alexey Suhov committed
296
        update_ie_fields(new_attrs, self.ir_version)
openvino-pushbot's avatar
openvino-pushbot committed
297 298 299
        self.substitute_ie_attrs(new_attrs)
        for k, v in new_attrs.items():
            node[k] = v
300
        node.update_node()
openvino-pushbot's avatar
openvino-pushbot committed
301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320

    @classmethod
    def update_node_stat(cls, node: Node, attrs: dict = None):
        if attrs is None:
            attrs = dict()
        op = cls(node.graph, attrs)
        op.update_node(node)

    def supported_attrs(self):
        """
        Attributes that user should/can set for the operation
        """
        return []

    def backend_attrs(self):
        """
        Attributes that will be translated to back-end IR
        """
        return self.supported_attrs()

Alexey Suhov's avatar
Alexey Suhov committed
321 322 323
    def backend_attrs_v2(self):
        return self.backend_attrs()

openvino-pushbot's avatar
openvino-pushbot committed
324 325 326 327 328 329 330 331 332 333 334 335 336 337
    @staticmethod
    def get_op_class_by_name(name: str):
        return __class__.registered_ops[name]

    @classmethod
    def class_type(cls):
        return class_registration.ClassType.OP

    @staticmethod
    def expand_node_shape(node: Node, dims_to_add):
        if node is None or not node.has_valid('value'):
            return
        for idx in range(dims_to_add):
            node.value = np.expand_dims(node.value, axis=-1)
Alexey Suhov's avatar
Alexey Suhov committed
338 339 340 341 342 343 344 345 346 347 348 349 350 351
        node.shape = np.array(node.value.shape)


class PermuteAttrs:
    Permutation = namedtuple('Permutation', ['perm', 'inv'])
    Attr = namedtuple('Attr', ['name', 'port', 'func'])

    common_permutation = lambda node, permutation, attr: node[attr][permutation.perm]
    common_permutation_inv = lambda node, permutation, attr: permutation.inv[node[attr]]

    # List of default permutations
    common_attrs_permutation = {
            'dim': common_permutation,
            'pad': common_permutation,
352
            'pads': common_permutation,
Alexey Suhov's avatar
Alexey Suhov committed
353 354 355 356 357 358 359 360 361 362
            'shape': common_permutation,
            'order': lambda node, permutation, attr: permutation.inv[node[attr][permutation.perm]],
            'stride': common_permutation,
            'window': common_permutation,
            'dilation': common_permutation,
            'kernel_shape': common_permutation,
            'output_shape': common_permutation,
            'slices': common_permutation,
            'shrink_axis_mask': common_permutation,
            'new_axis_mask': common_permutation,
363
            'axes': common_permutation_inv,
Alexey Suhov's avatar
Alexey Suhov committed
364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422
            'axis': common_permutation_inv,
            'batch_dims': common_permutation_inv,
            'channel_dims': common_permutation_inv,
            'spatial_dims': common_permutation_inv,

            'input_channel_dim': common_permutation_inv,
            'output_channel_dim': common_permutation_inv,
            'kernel_spatial_idx': common_permutation_inv,
            'input_feature_channel': common_permutation_inv,
            'output_feature_channel': common_permutation_inv,
    }

    @staticmethod
    def __attr(name, port, func=None):
        if func is None:
            if name in PermuteAttrs.common_attrs_permutation:
                func = PermuteAttrs.common_attrs_permutation[name]
            else:
                raise Error('Attr {} is missing in PermuteAttrs.common_attrs_permutation. Please update '
                            'common_attrs_permutation with permutation for your attribute!'.format(name))

        if len(port.split(':')) != 2 or port.split(':')[0] not in ['input', 'output']:
            raise Error("Attribute port {} for {} wasn't set correctly!".format(port, name))

        return PermuteAttrs.Attr(name=name, port=port, func=func)

    def __init__(self):
        self.attrs = {}

    def update_attrs(self, attrs):
        for attr in attrs:
            if not isinstance(attr, tuple) or len(attr) not in [2, 3]:
                raise Error('attr object must be a tuple: (attribute_name, port) or (attribute_name, port, func)')
            self.attrs.update({attr[0]: self.__attr(*attr)})
        return self

    def permute_attrs(self, node):
        # This function applies permutation for given node
        for attr in self.attrs.keys():
            name, port, func = self.attrs[attr]
            node_type, port = port.split(':')
            port = int(port)
            node_with_permutation = node.in_node(port) if node_type == 'input' else node.out_node(port)

            if node_with_permutation.has_valid('permutation'):
                permutation = node_with_permutation.permutation
                if isinstance(permutation, type(lambda: 0)):
                    node[name] = func(node, permutation(node), name)
                else:
                    node[name] = func(node, permutation, name)

    @staticmethod
    def create_permute_attrs(node, attrs=None):
        # Create permute_attrs if not exists
        if not node.has_valid('permute_attrs'):
            node['permute_attrs'] = PermuteAttrs()
        node['permute_attrs'].update_attrs(attrs)

    @staticmethod
423
    def set_permutation(node1, node2, permutation, override=False):
Alexey Suhov's avatar
Alexey Suhov committed
424 425
        # This function creates permutation on edge between node1->node2
        edge_attrs = node1.graph.get_edge_data(node1.id, node2.id)[0]
426
        if 'permutation' not in edge_attrs or override:
Alexey Suhov's avatar
Alexey Suhov committed
427 428 429 430
            nx.set_edge_attributes(G=node1.graph,
                                   values={(node1.id, node2.id, 0): permutation},
                                   name='permutation')
        else:
431 432 433
            # If permutation exists we check that given and already set permutations are equal
            if (edge_attrs['permutation'] is None and permutation is not None) or \
                not np.array_equal(edge_attrs['permutation'], permutation):
434
                raise Error('Permutation already exists in edge between {} and {}'.format(node1.id, node2.id))
Alexey Suhov's avatar
Alexey Suhov committed
435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466

    @staticmethod
    def get_inverse_permutation(perm):
        inv = [0] * len(perm)
        # Create reverse permutation
        for index, pos in enumerate(perm):
            inv[pos] = index
        return inv

    @staticmethod
    def get_nhwc_to_nchw_permutation(dims_number: int):
        # This function returns permutation from NHWC to NCHW for given dims number
        if dims_number != 3:
            perm = [0, dims_number - 1, *[x for x in range(1, dims_number - 1)]] if dims_number > 1 else [x for x in range(
                dims_number)]
        else:
            # Exclude 3D shapes from permutation process: identity permutation
            perm = list(range(0, dims_number))
        inv = PermuteAttrs.get_inverse_permutation(perm)
        return PermuteAttrs.Permutation(perm=np.array(perm), inv=np.array(inv))

    @staticmethod
    def get_nchw_to_nhwc_permutation(dims_number: int):
        # This function returns permutation from NCHW to NHWC for given dims number
        if dims_number != 3:
            perm = [0, *[x for x in range(2, dims_number)], 1] if dims_number > 1 else [x for x in range(
                dims_number)]
        else:
            # Exclude 3D shapes from permutation process: identity permutation
            perm = list(range(0, dims_number))
        inv = PermuteAttrs.get_inverse_permutation(perm)
        return PermuteAttrs.Permutation(perm=np.array(perm), inv=np.array(inv))