cli_parser.py 41.7 KB
Newer Older
openvino-pushbot's avatar
openvino-pushbot committed
1
"""
2
 Copyright (c) 2018-2019 Intel Corporation
openvino-pushbot's avatar
openvino-pushbot committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
"""

import argparse
import os
import re
import sys
from collections import OrderedDict
from itertools import zip_longest

import numpy as np

from mo.front.extractor import split_node_in_port
from mo.utils import import_extensions
from mo.utils.error import Error
from mo.utils.utils import refer_to_faq_msg


class CanonicalizePathAction(argparse.Action):
    """
    Expand user home directory paths and convert relative-paths to absolute.
    """
Alexey Suhov's avatar
Alexey Suhov committed
36

openvino-pushbot's avatar
openvino-pushbot committed
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55
    def __call__(self, parser, namespace, values, option_string=None):
        if values is not None:
            list_of_values = list()
            if isinstance(values, str):
                if values != "":
                    list_of_values = values.split(',')
            elif isinstance(values, list):
                list_of_values = values
            else:
                raise Error('Unsupported type of command line parameter "{}" value'.format(self.dest))
            list_of_values = [get_absolute_path(path) for path in list_of_values]
            setattr(namespace, self.dest, ','.join(list_of_values))


class CanonicalizePathCheckExistenceAction(CanonicalizePathAction):
    """
    Expand user home directory paths and convert relative-paths to absolute and check specified file or directory
    existence.
    """
Alexey Suhov's avatar
Alexey Suhov committed
56

openvino-pushbot's avatar
openvino-pushbot committed
57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
    def __call__(self, parser, namespace, values, option_string=None):
        super().__call__(parser, namespace, values, option_string)
        names = getattr(namespace, self.dest)
        for name in names.split(','):
            if name != "" and not os.path.exists(name):
                raise Error('The value for command line parameter "{}" must be existing file/directory, '
                            ' but "{}" does not exist.'.format(self.dest, name))


def readable_file(path: str):
    """
    Check that specified path is a readable file.
    :param path: path to check
    :return: path if the file is readable
    """
    if not os.path.isfile(path):
        raise Error('The "{}" is not existing file'.format(path))
    elif not os.access(path, os.R_OK):
        raise Error('The "{}" is not readable'.format(path))
    else:
        return path


def readable_dirs(paths: str):
    """
    Checks that comma separated list of paths are readable directories.
    :param paths: comma separated list of paths.
    :return: comma separated list of paths.
    """
    paths_list = [readable_dir(path) for path in paths.split(',')]
    return ','.join(paths_list)


def readable_dirs_or_empty(paths: str):
    """
    Checks that comma separated list of paths are readable directories of if it is empty.
    :param paths: comma separated list of paths.
    :return: comma separated list of paths.
    """
    if paths:
        return readable_dirs(paths)
    return paths


def readable_dir(path: str):
    """
    Check that specified path is a readable directory.
    :param path: path to check
    :return: path if the directory is readable
    """
    if not os.path.isdir(path):
        raise Error('The "{}" is not existing directory'.format(path))
    elif not os.access(path, os.R_OK):
        raise Error('The "{}" is not readable'.format(path))
    else:
        return path


def writable_dir(path: str):
    """
    Checks that specified directory is writable. The directory may not exist but it's parent or grandparent must exist.
    :param path: path to check that it is writable.
    :return: path if it is writable
    """
    if path is None:
        raise Error('The directory parameter is None')
    if os.path.exists(path):
        if os.path.isdir(path):
            if os.access(path, os.W_OK):
                return path
            else:
                raise Error('The directory "{}" is not writable'.format(path))
        else:
            raise Error('The "{}" is not a directory'.format(path))
    else:
        cur_path = path
        while os.path.dirname(cur_path) != cur_path:
            if os.path.exists(cur_path):
                break
            cur_path = os.path.dirname(cur_path)
        if cur_path == '':
            cur_path = os.path.curdir
        if os.access(cur_path, os.W_OK):
            return path
        else:
            raise Error('The directory "{}" is not writable'.format(cur_path))

144

openvino-pushbot's avatar
openvino-pushbot committed
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
def get_common_cli_parser(parser: argparse.ArgumentParser = None):
    if not parser:
        parser = argparse.ArgumentParser()
    common_group = parser.add_argument_group('Framework-agnostic parameters')
    # Common parameters
    common_group.add_argument('--input_model', '-w', '-m',
                              help='Tensorflow*: a file with a pre-trained model ' +
                                   ' (binary or text .pb file after freezing).\n' +
                                   ' Caffe*: a model proto file with model weights',
                              action=CanonicalizePathCheckExistenceAction,
                              type=readable_file)
    common_group.add_argument('--model_name', '-n',
                              help='Model_name parameter passed to the final create_ir transform. ' +
                                   'This parameter is used to name ' +
                                   'a network in a generated IR and output .xml/.bin files.')
    common_group.add_argument('--output_dir', '-o',
                              help='Directory that stores the generated IR. ' +
                                   'By default, it is the directory from where the Model Optimizer is launched.',
                              default=get_absolute_path('.'),
                              action=CanonicalizePathAction,
                              type=writable_dir)
    common_group.add_argument('--input_shape',
Alexey Suhov's avatar
Alexey Suhov committed
167 168 169 170 171 172 173 174 175 176 177
                              help='Input shape(s) that should be fed to an input node(s) of the model. '
                                   'Shape is defined as a comma-separated list of integer numbers enclosed in '
                                   'parentheses or square brackets, for example [1,3,227,227] or (1,227,227,3), where '
                                   'the order of dimensions depends on the framework input layout of the model. '
                                   'For example, [N,C,H,W] is used for Caffe* models and [N,H,W,C] for TensorFlow* '
                                   'models. Model Optimizer performs necessary transformations to convert the shape to '
                                   'the layout required by Inference Engine (N,C,H,W). The shape should not contain '
                                   'undefined dimensions (? or -1) and should fit the dimensions defined in the input '
                                   'operation of the graph. If there are multiple inputs in the model, --input_shape '
                                   'should contain definition of shape for each input separated by a comma, for '
                                   'example: [1,3,227,227],[2,4] for a model with two inputs with 4D and 2D shapes.')
openvino-pushbot's avatar
openvino-pushbot committed
178 179 180 181 182 183 184 185 186
    common_group.add_argument('--scale', '-s',
                              type=float,
                              help='All input values coming from original network inputs will be ' +
                                   'divided by this ' +
                                   'value. When a list of inputs is overridden by the --input ' +
                                   'parameter, this scale ' +
                                   'is not applied for any input that does not match with ' +
                                   'the original input of the model.')
    common_group.add_argument('--reverse_input_channels',
Alexey Suhov's avatar
Alexey Suhov committed
187 188 189 190 191
                              help='Switch the input channels order from RGB to BGR (or vice versa). Applied to '
                                   'original inputs of the model if and only if a number of channels equals 3. Applied '
                                   'after application of --mean_values and --scale_values options, so numbers in '
                                   '--mean_values and --scale_values go in the order of channels used in the original '
                                   'model.',
openvino-pushbot's avatar
openvino-pushbot committed
192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207
                              action='store_true')
    common_group.add_argument('--log_level',
                              help='Logger level',
                              choices=['CRITICAL', 'ERROR', 'WARN', 'WARNING', 'INFO',
                                       'DEBUG', 'NOTSET'],
                              default='ERROR')
    common_group.add_argument('--input',
                              help='The name of the input operation of the given model. ' +
                                   'Usually this is a name of the ' +
                                   'input placeholder of the model.')
    common_group.add_argument('--output',
                              help='The name of the output operation of the model. ' +
                                   'For TensorFlow*, do not add :0 to this name.')
    common_group.add_argument('--mean_values', '-ms',
                              help='Mean values to be used for the input image per channel. ' +
                                   'Values to be provided in the (R,G,B) or [R,G,B] format. ' +
Alexey Suhov's avatar
Alexey Suhov committed
208 209 210
                                   'Can be defined for desired input of the model, for example: ' +
                                   '"--mean_values data[255,255,255],info[255,255,255]". ' +
                                   'The exact meaning and order ' +
openvino-pushbot's avatar
openvino-pushbot committed
211 212 213 214
                                   'of channels depend on how the original model was trained.',
                              default=())
    common_group.add_argument('--scale_values',
                              help='Scale values to be used for the input image per channel. ' +
Alexey Suhov's avatar
Alexey Suhov committed
215 216 217
                                   'Values are provided in the (R,G,B) or [R,G,B] format. ' +
                                   'Can be defined for desired input of the model, for example: ' +
                                   '"--scale_values data[255,255,255],info[255,255,255]". ' +
openvino-pushbot's avatar
openvino-pushbot committed
218 219 220 221 222 223 224 225 226 227 228
                                   'The exact meaning and order ' +
                                   'of channels depend on how the original model was trained.',
                              default=())
    # TODO: isn't it a weights precision type
    common_group.add_argument('--data_type',
                              help='Data type for all intermediate tensors and weights. ' +
                                   'If original model is in FP32 and --data_type=FP16 is specified, all model weights ' +
                                   'and biases are quantized to FP16.',
                              choices=["FP16", "FP32", "half", "float"],
                              default='float')
    common_group.add_argument('--disable_fusing',
Alexey Suhov's avatar
Alexey Suhov committed
229
                              help='Turn off fusing of linear operations to Convolution',
openvino-pushbot's avatar
openvino-pushbot committed
230 231
                              action='store_true')
    common_group.add_argument('--disable_resnet_optimization',
Alexey Suhov's avatar
Alexey Suhov committed
232
                              help='Turn off resnet optimization',
openvino-pushbot's avatar
openvino-pushbot committed
233 234 235 236 237
                              action='store_true')
    common_group.add_argument('--finegrain_fusing',
                              help='Regex for layers/operations that won\'t be fused. ' +
                                   'Example: --finegrain_fusing Convolution1,.*Scale.*')
    common_group.add_argument('--disable_gfusing',
Alexey Suhov's avatar
Alexey Suhov committed
238
                              help='Turn off fusing of grouped convolutions',
openvino-pushbot's avatar
openvino-pushbot committed
239
                              action='store_true')
240 241 242
    common_group.add_argument('--enable_concat_optimization',
                              help='Turn on concat optimization',
                              action='store_true')
openvino-pushbot's avatar
openvino-pushbot committed
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
    common_group.add_argument('--move_to_preprocess',
                              help='Move mean values to IR preprocess section',
                              action='store_true')
    # we use CanonicalizeDirCheckExistenceAction instead of readable_dirs to handle empty strings
    common_group.add_argument("--extensions",
                              help="Directory or a comma separated list of directories with extensions. To disable all "
                                   "extensions including those that are placed at the default location, pass an empty "
                                   "string.",
                              default=import_extensions.default_path(),
                              action=CanonicalizePathCheckExistenceAction,
                              type=readable_dirs_or_empty)
    common_group.add_argument("--batch", "-b",
                              type=check_positive,
                              default=None,
                              help="Input batch size")
    common_group.add_argument("--version",
                              action='store_true',
                              help="Version of Model Optimizer")

    common_group.add_argument('--silent',
Alexey Suhov's avatar
Alexey Suhov committed
263
                              help='Prevent any output messages except those that correspond to log level equals '
openvino-pushbot's avatar
openvino-pushbot committed
264 265 266 267
                                   'ERROR, that can be set with the following option: --log_level. '
                                   'By default, log level is already ERROR. ',
                              action='store_true',
                              default=False)
Alexey Suhov's avatar
Alexey Suhov committed
268 269
    common_group.add_argument('--freeze_placeholder_with_value', help='Replaces input layer with constant node with '
                                                                      'provided value, e.g.: "node_name->True"',
openvino-pushbot's avatar
openvino-pushbot committed
270
                              default=None)
Alexey Suhov's avatar
Alexey Suhov committed
271 272 273 274 275 276 277 278
    common_group.add_argument('--generate_deprecated_IR_V2',
                              help='Force to generate legacy/deprecated IR V2 to work with previous versions of the'
                                   ' Inference Engine. The resulting IR may or may not be correctly loaded by'
                                   ' Inference Engine API (including the most recent and old versions of Inference'
                                   ' Engine) and provided as a partially-validated backup option for specific'
                                   ' deployment scenarios. Use it at your own discretion. By default, without this'
                                   ' option, the Model Optimizer generates IR V3.',
                              action='store_true')
279 280 281 282
    common_group.add_argument('--keep_shape_ops',
                              help='[ Experimental feature ] Enables `Shape` operation with all children keeping. '
                                   'This feature makes model reshapable in Inference Engine',
                              action='store_true', default=False)
openvino-pushbot's avatar
openvino-pushbot committed
283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327
    return parser


def get_common_cli_options(model_name):
    d = OrderedDict()
    d['input_model'] = '- Path to the Input Model'
    d['output_dir'] = ['- Path for generated IR', lambda x: x if x != '.' else os.getcwd()]
    d['model_name'] = ['- IR output name', lambda x: x if x else model_name]
    d['log_level'] = '- Log level'
    d['batch'] = ['- Batch', lambda x: x if x else 'Not specified, inherited from the model']
    d['input'] = ['- Input layers', lambda x: x if x else 'Not specified, inherited from the model']
    d['output'] = ['- Output layers', lambda x: x if x else 'Not specified, inherited from the model']
    d['input_shape'] = ['- Input shapes', lambda x: x if x else 'Not specified, inherited from the model']
    d['mean_values'] = ['- Mean values', lambda x: x if x else 'Not specified']
    d['scale_values'] = ['- Scale values', lambda x: x if x else 'Not specified']
    d['scale'] = ['- Scale factor', lambda x: x if x else 'Not specified']
    d['data_type'] = ['- Precision of IR', lambda x: 'FP32' if x == 'float' else 'FP16' if x == 'half' else x]
    d['disable_fusing'] = ['- Enable fusing', lambda x: not x]
    d['disable_gfusing'] = ['- Enable grouped convolutions fusing', lambda x: not x]
    d['move_to_preprocess'] = '- Move mean values to preprocess section'
    d['reverse_input_channels'] = '- Reverse input channels'
    return d


def get_caffe_cli_options():
    d = {
        'input_proto': ['- Path to the Input prototxt', lambda x: x],
        'mean_file': ['- Path to a mean file', lambda x: x if x else 'Not specified'],
        'mean_file_offsets': ['- Offsets for a mean file', lambda x: x if x else 'Not specified'],
        'k': '- Path to CustomLayersMapping.xml',
        'disable_resnet_optimization': ['- Enable resnet optimization', lambda x: not x],
    }

    return OrderedDict(sorted(d.items(), key=lambda t: t[0]))


def get_tf_cli_options():
    d = {
        'input_model_is_text': '- Input model in text protobuf format',
        'tensorflow_subgraph_patterns': '- Patterns to offload',
        'tensorflow_operation_patterns': '- Operations to offload',
        'tensorflow_custom_operations_config_update': '- Update the configuration file with input/output node names',
        'tensorflow_use_custom_operations_config': '- Use the config file',
        'tensorflow_object_detection_api_pipeline_config': '- Use configuration file used to generate the model with '
                                                           'Object Detection API',
Alexey Suhov's avatar
Alexey Suhov committed
328
        'tensorflow_custom_layer_libraries': '- List of shared libraries with TensorFlow custom layers implementation',
openvino-pushbot's avatar
openvino-pushbot committed
329 330 331 332 333 334 335 336
        'tensorboard_logdir': '- Path to model dump for TensorBoard'
    }

    return OrderedDict(sorted(d.items(), key=lambda t: t[0]))


def get_mxnet_cli_options():
    d = {
337
        'input_symbol': '- Deploy-ready symbol file',
openvino-pushbot's avatar
openvino-pushbot committed
338
        'nd_prefix_name': '- Prefix name for args.nd and argx.nd files',
339 340 341
        'pretrained_model_name': '- Pretrained model to be merged with the .nd files',
        'save_params_from_nd': '- Enable saving built parameters file from .nd files',
        'legacy_mxnet_model': '- Enable MXNet loader for models trained with MXNet version lower than 1.0.0'
openvino-pushbot's avatar
openvino-pushbot committed
342 343 344 345 346 347 348
    }

    return OrderedDict(sorted(d.items(), key=lambda t: t[0]))


def get_kaldi_cli_options():
    d = {
349 350
        'counts': '- A file name with full path to the counts file',
        'remove_output_softmax': '- Removes the SoftMax layer that is the output layer'
openvino-pushbot's avatar
openvino-pushbot committed
351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389
    }

    return OrderedDict(sorted(d.items(), key=lambda t: t[0]))


def get_onnx_cli_options():
    d = {
    }

    return OrderedDict(sorted(d.items(), key=lambda t: t[0]))


def get_caffe_cli_parser(parser: argparse.ArgumentParser = None):
    """
    Specifies cli arguments for Model Optimizer for Caffe*

    Returns
    -------
        ArgumentParser instance
    """
    if not parser:
        parser = argparse.ArgumentParser()
        get_common_cli_parser(parser=parser)

    caffe_group = parser.add_argument_group('Caffe*-specific parameters')

    caffe_group.add_argument('--input_proto', '-d',
                             help='Deploy-ready prototxt file that contains a topology structure ' +
                                  'and layer attributes',
                             type=str,
                             action=CanonicalizePathCheckExistenceAction)
    caffe_group.add_argument('-k',
                             help='Path to CustomLayersMapping.xml to register custom layers',
                             type=str,
                             default=os.path.join(os.path.dirname(sys.argv[0]), 'extensions', 'front', 'caffe',
                                                  'CustomLayersMapping.xml'),
                             action=CanonicalizePathCheckExistenceAction)
    caffe_group.add_argument('--mean_file', '-mf',
                             help='Mean image to be used for the input. Should be a binaryproto file',
Alexey Suhov's avatar
Alexey Suhov committed
390
                             default=None,
openvino-pushbot's avatar
openvino-pushbot committed
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428
                             action=CanonicalizePathCheckExistenceAction)
    caffe_group.add_argument('--mean_file_offsets', '-mo',
                             help='Mean image offsets to be used for the input binaryproto file. ' +
                                  'When the mean image is bigger than the expected input, it is cropped. By default, centers ' +
                                  'of the input image and the mean image are the same and the mean image is cropped by ' +
                                  'dimensions of the input image. The format to pass this option is the following: "-mo (x,y)". In this ' +
                                  'case, the mean file is cropped by dimensions of the input image with offset (x,y) ' +
                                  'from the upper left corner of the mean image',
                             default=None)
    caffe_group.add_argument('--disable_omitting_optional',
                             help='Disable omitting optional attributes to be used for custom layers. ' +
                                  'Use this option if you want to transfer all attributes of a custom layer to IR. ' +
                                  'Default behavior is to transfer the attributes with default values and the attributes defined by the user to IR.',
                             action='store_true',
                             default=False)
    caffe_group.add_argument('--enable_flattening_nested_params',
                             help='Enable flattening optional params to be used for custom layers. ' +
                                  'Use this option if you want to transfer attributes of a custom layer to IR with flattened nested parameters. ' +
                                  'Default behavior is to transfer the attributes without flattening nested parameters.',
                             action='store_true',
                             default=False)
    return parser


def get_tf_cli_parser(parser: argparse.ArgumentParser = None):
    """
    Specifies cli arguments for Model Optimizer for TF

    Returns
    -------
        ArgumentParser instance
    """
    if not parser:
        parser = argparse.ArgumentParser()
        get_common_cli_parser(parser=parser)

    tf_group = parser.add_argument_group('TensorFlow*-specific parameters')
    tf_group.add_argument('--input_model_is_text',
Alexey Suhov's avatar
Alexey Suhov committed
429 430
                          help='TensorFlow*: treat the input model file as a text protobuf format. If not specified, ' +
                               'the Model Optimizer treats it as a binary file by default.',
openvino-pushbot's avatar
openvino-pushbot committed
431
                          action='store_true')
Alexey Suhov's avatar
Alexey Suhov committed
432
    tf_group.add_argument('--input_checkpoint', type=str, default=None, help="TensorFlow*: variables file to load.",
openvino-pushbot's avatar
openvino-pushbot committed
433 434
                          action=CanonicalizePathCheckExistenceAction)
    tf_group.add_argument('--input_meta_graph',
Alexey Suhov's avatar
Alexey Suhov committed
435
                          help='Tensorflow*: a file with a meta-graph of the model before freezing',
openvino-pushbot's avatar
openvino-pushbot committed
436 437
                          action=CanonicalizePathCheckExistenceAction,
                          type=readable_file)
Alexey Suhov's avatar
Alexey Suhov committed
438 439
    tf_group.add_argument('--saved_model_dir', default=None,
                          help="TensorFlow*: directory representing non frozen model",
openvino-pushbot's avatar
openvino-pushbot committed
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
                          action=CanonicalizePathCheckExistenceAction,
                          type=readable_dirs)
    tf_group.add_argument('--saved_model_tags', type=str, default=None,
                          help="Group of tag(s) of the MetaGraphDef to load, in string format, separated by ','. "
                               "For tag-set contains multiple tags, all tags must be passed in.")
    tf_group.add_argument('--tensorflow_subgraph_patterns',
                          help='TensorFlow*: a list of comma separated patterns that will be applied to ' +
                               'TensorFlow* node names to ' +
                               'infer a part of the graph using TensorFlow*.')
    tf_group.add_argument('--tensorflow_operation_patterns',
                          help='TensorFlow*: a list of comma separated patterns that will be applied to ' +
                               'TensorFlow* node type (ops) ' +
                               'to infer these operations using TensorFlow*.')
    tf_group.add_argument('--tensorflow_custom_operations_config_update',
                          help='TensorFlow*: update the configuration file with node name patterns with input/output '
                               'nodes information.',
                          action=CanonicalizePathCheckExistenceAction)
    tf_group.add_argument('--tensorflow_use_custom_operations_config',
                          help='TensorFlow*: use the configuration file with custom operation description.',
                          action=CanonicalizePathCheckExistenceAction)
    tf_group.add_argument('--tensorflow_object_detection_api_pipeline_config',
                          help='TensorFlow*: path to the pipeline configuration file used to generate model created '
                               'with help of Object Detection API.',
                          action=CanonicalizePathCheckExistenceAction)
    tf_group.add_argument('--tensorboard_logdir',
                          help='TensorFlow*: dump the input graph to a given directory that should be used with TensorBoard.',
                          default=None,
                          action=CanonicalizePathCheckExistenceAction)
Alexey Suhov's avatar
Alexey Suhov committed
468 469 470 471 472 473 474 475
    tf_group.add_argument('--tensorflow_custom_layer_libraries',
                          help='TensorFlow*: comma separated list of shared libraries with TensorFlow* custom '
                               'operations implementation.',
                          default=None,
                          action=CanonicalizePathCheckExistenceAction)
    tf_group.add_argument('--disable_nhwc_to_nchw',
                          help='Disables default translation from NHWC to NCHW',
                          action='store_true')
openvino-pushbot's avatar
openvino-pushbot committed
476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
    return parser


def get_mxnet_cli_parser(parser: argparse.ArgumentParser = None):
    """
    Specifies cli arguments for Model Optimizer for MXNet*

    Returns
    -------
        ArgumentParser instance
    """
    if not parser:
        parser = argparse.ArgumentParser()
        get_common_cli_parser(parser=parser)

    mx_group = parser.add_argument_group('Mxnet-specific parameters')

    mx_group.add_argument('--input_symbol',
                          help='Symbol file (for example, model-symbol.json) that contains a topology structure ' +
Alexey Suhov's avatar
Alexey Suhov committed
495
                               'and layer attributes',
openvino-pushbot's avatar
openvino-pushbot committed
496 497 498 499
                          type=str,
                          action=CanonicalizePathCheckExistenceAction)
    mx_group.add_argument("--nd_prefix_name",
                          help="Prefix name for args.nd and argx.nd files.",
Alexey Suhov's avatar
Alexey Suhov committed
500
                          default=None)
openvino-pushbot's avatar
openvino-pushbot committed
501
    mx_group.add_argument("--pretrained_model_name",
502
                          help="Name of a pretrained MXNet model without extension and epoch number. This model will be merged with args.nd and argx.nd files",
Alexey Suhov's avatar
Alexey Suhov committed
503
                          default=None)
openvino-pushbot's avatar
openvino-pushbot committed
504 505
    mx_group.add_argument("--save_params_from_nd",
                          action='store_true',
506
                          help="Enable saving built parameters file from .nd files")
openvino-pushbot's avatar
openvino-pushbot committed
507 508
    mx_group.add_argument("--legacy_mxnet_model",
                          action='store_true',
509
                          help="Enable MXNet loader to make a model compatible with the latest MXNet version. Use only if your model was trained with MXNet version lower than 1.0.0")
openvino-pushbot's avatar
openvino-pushbot committed
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
    return parser


def get_kaldi_cli_parser(parser: argparse.ArgumentParser = None):
    """
    Specifies cli arguments for Model Optimizer for MXNet*

    Returns
    -------
        ArgumentParser instance
    """
    if not parser:
        parser = argparse.ArgumentParser()
        get_common_cli_parser(parser=parser)

Alexey Suhov's avatar
Alexey Suhov committed
525
    kaldi_group = parser.add_argument_group('Kaldi-specific parameters')
openvino-pushbot's avatar
openvino-pushbot committed
526

Alexey Suhov's avatar
Alexey Suhov committed
527 528 529 530
    kaldi_group.add_argument("--counts",
                             help="Path to the counts file",
                             default=None,
                             action=CanonicalizePathCheckExistenceAction)
openvino-pushbot's avatar
openvino-pushbot committed
531

Alexey Suhov's avatar
Alexey Suhov committed
532
    kaldi_group.add_argument("--remove_output_softmax",
533
                             help="Removes the SoftMax layer that is the output layer",
Alexey Suhov's avatar
Alexey Suhov committed
534
                             action='store_true')
openvino-pushbot's avatar
openvino-pushbot committed
535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670
    return parser


def get_onnx_cli_parser(parser: argparse.ArgumentParser = None):
    """
    Specifies cli arguments for Model Optimizer for ONNX

    Returns
    -------
        ArgumentParser instance
    """
    if not parser:
        parser = argparse.ArgumentParser()
        get_common_cli_parser(parser=parser)

    tf_group = parser.add_argument_group('ONNX*-specific parameters')

    return parser


def get_all_cli_parser():
    """
    Specifies cli arguments for Model Optimizer

    Returns
    -------
        ArgumentParser instance
    """
    parser = argparse.ArgumentParser()

    parser.add_argument('--framework',
                        help='Name of the framework used to train the input model.',
                        type=str,
                        choices=['tf', 'caffe', 'mxnet', 'kaldi', 'onnx'])

    get_common_cli_parser(parser=parser)

    get_tf_cli_parser(parser=parser)
    get_caffe_cli_parser(parser=parser)
    get_mxnet_cli_parser(parser=parser)
    get_kaldi_cli_parser(parser=parser)
    get_onnx_cli_parser(parser=parser)

    return parser


def get_placeholder_shapes(argv_input: str, argv_input_shape: str, argv_batch=None):
    """
    Parses input layers names and input shapes from the cli and returns the parsed object

    Parameters
    ----------
    argv_input
        string with a list of input layers: either an empty string, or strings separated with comma.
        E.g. 'inp1,inp2'
    argv_input_shape
        string with a list of input shapes: either an empty string, or tuples separated with comma.
        E.g. '(1,2),(3,4)'.
        Only positive integers are accepted except -1, which can be on any position in a shape.
    argv_batch
        integer that overrides batch size in input shape

    Returns
    -------
        parsed shapes in form of {'name of input':ndarray} if names of inputs are provided with shapes
        parsed shapes in form of {'name of input':None} if names of inputs are provided without shapes
        ndarray if only one shape is provided and no input name
        None if neither shape nor input were provided
    """
    if argv_input_shape and argv_batch:
        raise Error("Both --input_shape and --batch were provided. Please provide only one of them. " +
                    refer_to_faq_msg(56))
    shapes = list()
    inputs = list()
    placeholder_shapes = None

    first_digit_reg = r'([0-9 ]+|-1)'
    next_digits_reg = r'(,{})*'.format(first_digit_reg)
    tuple_reg = r'((\({}{}\))|(\[{}{}\]))'.format(first_digit_reg, next_digits_reg,
                                                  first_digit_reg, next_digits_reg)
    if argv_input_shape:
        full_reg = r'^{}(\s*,\s*{})*$|^$'.format(tuple_reg, tuple_reg)
        if not re.match(full_reg, argv_input_shape):
            raise Error('Input shape "{}" cannot be parsed. ' +
                        refer_to_faq_msg(57), argv_input_shape)

        shapes = re.findall(r'[(\[]([0-9, -]+)[)\]]', argv_input_shape)

    if argv_input:
        inputs = argv_input.split(',')

    # check number of shapes with no input provided
    if argv_input_shape and not argv_input:
        if len(shapes) > 1:
            raise Error('Please provide input layer names for input layer shapes. ' +
                        refer_to_faq_msg(58))
        else:
            placeholder_shapes = np.fromstring(shapes[0], dtype=np.int64, sep=',')

    # check if number of shapes does not match number of passed inputs
    elif argv_input and (len(shapes) == len(inputs) or len(shapes) == 0):
        placeholder_shapes = dict(zip_longest(inputs,
                                              map(lambda x: np.fromstring(x, dtype=np.int64,
                                                                          sep=',') if x else None, shapes)))
    elif argv_input:
        raise Error('Please provide each input layers with an input layer shape. ' +
                    refer_to_faq_msg(58))

    return placeholder_shapes


def parse_tuple_pairs(argv_values: str):
    """
    Gets mean/scale values from the given string parameter
    Parameters
    ----------
    argv_values
        string with a specified input name and  list of mean values: either an empty string, or a tuple
        in a form [] or ().
        E.g. 'data(1,2,3)' means 1 for the RED channel, 2 for the GREEN channel, 3 for the BLUE channel for the data
        input layer, or tuple of values in a form [] or () if input is specified separately, e.g. (1,2,3),[4,5,6].

    Returns
    -------
        dictionary with input name and tuple of values or list of values if mean/scale value is specified with input,
        e.g.:
        "data(10,20,30),info(11,22,33)" -> { 'data': [10,20,30], 'info': [11,22,33] }
        "(10,20,30),(11,22,33)" -> [np.array(10,20,30), np.array(11,22,33)]
    """
    res = {}
    if not argv_values:
        return res

    data_str = argv_values
    while True:
        tuples_matches = re.findall(r'[(\[]([0-9., -]+)[)\]]', data_str, re.IGNORECASE)
Alexey Suhov's avatar
Alexey Suhov committed
671 672 673 674 675
        if not tuples_matches :
            raise Error(
                "Mean/scale values should be in format: data(1,2,3),info(2,3,4)" +
                " or just plain set of them without naming any inputs: (1,2,3),(2,3,4). " +
                refer_to_faq_msg(101), argv_values)
openvino-pushbot's avatar
openvino-pushbot committed
676 677 678 679 680 681 682 683 684 685 686 687 688
        tuple_value = tuples_matches[0]
        matches = data_str.split(tuple_value)

        input_name = matches[0][:-1]
        if not input_name:
            res = []
            # check that other values are specified w/o names
            words_reg = r'([a-zA-Z]+)'
            for i in range(0, len(matches)):
                if re.search(words_reg, matches[i]) is not None:
                    # error - tuple with name is also specified
                    raise Error(
                        "Mean/scale values should either contain names of input layers: data(1,2,3),info(2,3,4)" +
Alexey Suhov's avatar
Alexey Suhov committed
689 690
                        " or just plain set of them without naming any inputs: (1,2,3),(2,3,4)." +
                        refer_to_faq_msg(101), argv_values)
openvino-pushbot's avatar
openvino-pushbot committed
691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894
            for match in tuples_matches:
                res.append(np.fromstring(match, dtype=float, sep=','))
            break

        res[input_name] = np.fromstring(tuple_value, dtype=float, sep=',')

        parenthesis = matches[0][-1]
        sibling = ')' if parenthesis == '(' else ']'
        pair = '{}{}{}{}'.format(input_name, parenthesis, tuple_value, sibling)
        idx_substr = data_str.index(pair)
        data_str = data_str[idx_substr + len(pair) + 1:]

        if not data_str:
            break

    return res


def get_tuple_values(argv_values: str or tuple, num_exp_values: int = 3, t=float or int):
    """
    Gets mean values from the given string parameter
    Args:
        argv_values: string with list of mean values: either an empty string, or a tuple in a form [] or ().
        E.g. '(1,2,3)' means 1 for the RED channel, 2 for the GREEN channel, 4 for the BLUE channel.
        t: either float or int
        num_exp_values: number of values in tuple

    Returns:
        tuple of values
    """

    digit_reg = r'(-?[0-9. ]+)' if t == float else r'(-?[0-9 ]+)'

    assert num_exp_values > 1, 'Can not parse tuple of size 1'
    content = r'{0}\s*,{1}\s*{0}'.format(digit_reg, (digit_reg + ',') * (num_exp_values - 2))
    tuple_reg = r'((\({0}\))|(\[{0}\]))'.format(content)

    if isinstance(argv_values, tuple) and not len(argv_values):
        return argv_values

    if not len(argv_values) or not re.match(tuple_reg, argv_values):
        raise Error('Values "{}" cannot be parsed. ' +
                    refer_to_faq_msg(59), argv_values)

    mean_values_matches = re.findall(r'[(\[]([0-9., -]+)[)\]]', argv_values)

    for mean in mean_values_matches:
        if len(mean.split(',')) != num_exp_values:
            raise Error('{} channels are expected for given values. ' +
                        refer_to_faq_msg(60), num_exp_values)

    return mean_values_matches


def get_mean_scale_dictionary(mean_values, scale_values, argv_input: str):
    """
    This function takes mean_values and scale_values, checks and processes them into convenient structure

    Parameters
    ----------
    mean_values dictionary, contains input name and mean values passed py user (e.g. {data: np.array[102.4, 122.1, 113.9]}),
    or list containing values (e.g. np.array[102.4, 122.1, 113.9])
    scale_values dictionary, contains input name and scale values passed py user (e.g. {data: np.array[102.4, 122.1, 113.9]})
    or list containing values (e.g. np.array[102.4, 122.1, 113.9])

    Returns
    -------
    The function returns a dictionary e.g.
    mean = { 'data: np.array, 'info': np.array }, scale = { 'data: np.array, 'info': np.array }, input = "data, info" ->
     { 'data': { 'mean': np.array, 'scale': np.array }, 'info': { 'mean': np.array, 'scale': np.array } }

    """
    res = {}
    # collect input names
    if argv_input:
        inputs = argv_input.split(',')
    else:
        inputs = []
        if type(mean_values) is dict:
            inputs = list(mean_values.keys())
        if type(scale_values) is dict:
            for name in scale_values.keys():
                if name not in inputs:
                    inputs.append(name)

    # create unified object containing both mean and scale for input
    if type(mean_values) is dict and type(scale_values) is dict:
        if not mean_values and not scale_values:
            return res
        for inp in inputs:
            inp, port = split_node_in_port(inp)
            if inp in mean_values or inp in scale_values:
                res.update(
                    {
                        inp: {
                            'mean':
                                mean_values[inp] if inp in mean_values else None,
                            'scale':
                                scale_values[inp] if inp in scale_values else None
                        }
                    }
                )
        return res

    # user specified input and mean/scale separately - we should return dictionary
    if inputs:
        if mean_values and scale_values:
            if len(inputs) != len(mean_values):
                raise Error('Numbers of inputs and mean values do not match. ' +
                            refer_to_faq_msg(61))
            if len(inputs) != len(scale_values):
                raise Error('Numbers of inputs and scale values do not match. ' +
                            refer_to_faq_msg(62))

            data = list(zip(mean_values, scale_values))

            for i in range(len(data)):
                res.update(
                    {
                        inputs[i]: {
                            'mean':
                                data[i][0],
                            'scale':
                                data[i][1],

                        }
                    }
                )
            return res
        # only mean value specified
        if mean_values:
            data = list(mean_values)
            for i in range(len(data)):
                res.update(
                    {
                        inputs[i]: {
                            'mean':
                                data[i],
                            'scale':
                                None

                        }
                    }
                )
            return res

        # only scale value specified
        if scale_values:
            data = list(scale_values)
            for i in range(len(data)):
                res.update(
                    {
                        inputs[i]: {
                            'mean':
                                None,
                            'scale':
                                data[i]

                        }
                    }
                )
            return res
    # mean and scale are specified without inputs, return list, order is not guaranteed (?)
    return list(zip_longest(mean_values, scale_values))


def get_model_name(path_input_model: str) -> str:
    """
    Deduces model name by a given path to the input model
    Args:
        path_input_model: path to the input model

    Returns:
        name of the output IR
    """
    parsed_name, extension = os.path.splitext(os.path.basename(path_input_model))
    return 'model' if parsed_name.startswith('.') or len(parsed_name) == 0 else parsed_name


def get_absolute_path(path_to_file: str) -> str:
    """
    Deduces absolute path of the file by a given path to the file
    Args:
        path_to_file: path to the file

    Returns:
        absolute path of the file
    """
    file_path = os.path.expanduser(path_to_file)
    if not os.path.isabs(file_path):
        file_path = os.path.join(os.getcwd(), file_path)
    return file_path


def check_positive(value):
    try:
        int_value = int(value)
        if int_value <= 0:
            raise ValueError
    except ValueError:
        raise argparse.ArgumentTypeError("expected a positive integer value")

    return int_value

Alexey Suhov's avatar
Alexey Suhov committed
895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923

def depersonalize(value: str):
    if not isinstance(value, str):
        return value
    res = []
    for path in value.split(','):
        if os.path.isdir(path):
            res.append('DIR')
        elif os.path.isfile(path):
            res.append(os.path.join('DIR', os.path.split(path)[1]))
        else:
            res.append(path)
    return ','.join(res)


def get_meta_info(argv: argparse.Namespace):
    meta_data = {'unset': []}
    for key, value in argv.__dict__.items():
        if value is not None:
            value = depersonalize(value)
            meta_data[key] = value
        else:
            meta_data['unset'].append(key)
    # The attribute 'k' is treated separately because it points to not existing file by default
    for key in ['k']:
        if key in meta_data:
            meta_data[key] = ','.join([os.path.join('DIR', os.path.split(i)[1]) for i in meta_data[key].split(',')])
    return meta_data