unit_test_executable.py 5.11 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140
import logging as log
import sys
import os
import csv
import pytest
import re
from conftest import shell

log.basicConfig(format="[ %(levelname)s ]  %(msg)s", stream=sys.stdout, level=log.INFO)

pytest.operation_dictionary = {}
pytest.avaliable_plugins = []


def save_coverage_to_csv(csv_path, header):
    with open(csv_path, 'w', newline='') as f:
        csv_writer = csv.writer(f, delimiter='|', quotechar='|', quoting=csv.QUOTE_MINIMAL)
        csv_writer.writerow(i for i in header)
        i = 1
        for key in sorted(pytest.operation_dictionary):
            line = [i, key]
            for plugin in pytest.avaliable_plugins:
                if not plugin in pytest.operation_dictionary[key]:
                    line.append('0/0')
                else:
                    line.append('/'.join(str(x) for x in pytest.operation_dictionary[key][plugin]))
            csv_writer.writerow(line)
            i += 1


def get_color(value):
    if '/' in value:
        passed, total = [int(x.strip()) for x in value.split('/')]
        if passed == total and total != 0:
            return "#d1ffd3"
        elif passed == total and total == 0:
            return "#dadada"
        else:
            return "#ffdbdb"
    else:
        return "white"


def csv_to_html_table(csv_path, html_path, headers=None, delimiter=","):
    with open(csv_path) as f:
        content = f.readlines()

    # reading file content into list
    rows = [x.strip() for x in content]
    table = "<!DOCTYPE html><html><head><title>Opset1 operations results</title></head><body><table border=1>"

    # creating HTML header row if header is provided
    if headers is not None:
        table += "<tr>"
        table += "".join(["<th>" + cell + "</th>" for cell in headers])
        table += "</tr>"
    else:
        table += "<tr>"
        table += "".join(["<th>" + cell + "</th>" for cell in rows[0].split(delimiter)])
        table += "</tr>"
        rows = rows[1:]

    # Converting csv to html row by row
    for row in rows:
        table += "<tr>" + "".join(["<td style=background-color:%s>" % (get_color(cell)) + cell + "</td>"
                                   for cell in row.split(delimiter)]) + "</tr>" + "\n"
    table += "</table></body></html><br>"

    # Saving html file
    fh = open(html_path, "w")
    fh.write(table)
    fh.close()


def setup_module():
    try:
        os.environ.get('PATH_TO_EXE')
    except KeyError:
        raise ImportError('PATH_TO_EXE is upsent in your environment variables. '
                          'Please, do "export PATH_TO_EXE=<path to unit-test>')


def teardown_module():
    """
    Creating CSV file at the end of test with nGraph nodes coverage
    :return:
    """
    csv_path = "nodes_coverage.csv"
    header = ["#", "Operation"] + [p + " passed / total" for p in pytest.avaliable_plugins]
    save_coverage_to_csv(csv_path=csv_path, header=header)

    # Convert csv file to html for better visualization
    html_path = "nodes_coverage.html"
    csv_to_html_table(csv_path=csv_path, html_path=html_path, delimiter="|")


def test(gtest_filter):
    executable = os.path.join(os.environ.get('PATH_TO_EXE'), "unit-test")
    cmd_line = executable + ' --gtest_filter=' + gtest_filter
    retcode, stdout = shell(cmd=cmd_line)

    # Parsing output of single test
    stdout = stdout.split('\n')
    nodes_list = []
    for line in stdout:
        if 'UNSUPPORTED OPS DETECTED!' in line:
            pytest.skip('Skip from pytest because unit-test send error UNSUPPORTED OPS DETECTED!')
        elif 'Nodes in test:' in line:
            nodes_list = list(set(line.replace('Nodes in test:', '').strip().split(' ')))

    if not nodes_list:
        pytest.skip('Skip from pytest because inside test no one ngraph function created')

    # Added one more loop, because condition below must be executed only if some nodes_list found
    # (it means that test includes opset1 operations)
    for line in stdout:
        if re.match('.*1 test from\s([A-Z]+)', line):
            matches = re.search(r'.*1 test from\s([A-Z]+)', line)
            plugin = matches.group(1)
            if plugin not in pytest.avaliable_plugins:
                pytest.avaliable_plugins.append(plugin)

    # Filling dictionary with operation coverage
    # How many time one operation is tested
    for n in nodes_list:
        if plugin in pytest.operation_dictionary[n]:
            numerator, denominator = pytest.operation_dictionary[n][plugin]
            pytest.operation_dictionary[n][plugin] = (numerator if retcode != 0 else numerator + 1,
                                                      denominator + 1)
        else:
            pytest.operation_dictionary[n][plugin] = (0, 1) if retcode != 0 else (1, 1)

    # This check is at the end, because with 99% it will return 0 or 1 (when function check of test failed)
    # Because the same cmd line executed by pytest_generate_tests with --gtest_list_tests.
    # So, most of the issue cached there.
    assert retcode == 0, "unit-test execution failed. Gtest failed. Return code: {}".format(retcode)


if __name__ == '__main__':
    log.warning("Please run {} by pytest like so:\npytest {} --gtest_filter=<attributes for gtest_filter>")