Skip to content

O
opencv
Commit 8ba88892 authored by Maksim Shabunin's avatar Maksim Shabunin
Browse files
Options

run.py: simplified scripts, fixed most of PEP8 warnings

parent c89ae6e5
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 84 additions and 142 deletions
modules/ts/misc/run.py
View file @ 8ba88892
#!/usr/bin/env python
import os, sys
import os
import argparse
import logging
import datetime
from run_utils import Err, CMakeCache, log, execute
from run_suite import TestSuite
from run_android import AndroidTestSuite
......@@ -21,13 +22,12 @@ if __name__ == "__main__":
description='OpenCV test runner script',
epilog=epilog,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("build_path", nargs = "*", default = ["."], help="Path to build directory (should contain CMakeCache.txt, default is current) or to directory with tests (all platform checks will be disabled in this case)")
parser.add_argument("build_path", nargs='?', default=".", help="Path to build directory (should contain CMakeCache.txt, default is current) or to directory with tests (all platform checks will be disabled in this case)")
parser.add_argument("-t", "--tests", metavar="MODULES", default="", help="Comma-separated list of modules to test (example: -t core,imgproc,java)")
parser.add_argument("-b", "--blacklist", metavar="MODULES", default="", help="Comma-separated list of modules to exclude from test (example: -b java)")
parser.add_argument("-a", "--accuracy", action="store_true", default=False, help="Look for accuracy tests instead of performance tests")
parser.add_argument("--check", action="store_true", default=False, help="Shortcut for '--perf_min_samples=1 --perf_force_samples=1'")
parser.add_argument("-w", "--cwd", metavar="PATH", default=".", help="Working directory for tests (default is current)")
parser.add_argument("-l", "--longname", action="store_true", default=False, help="Generate log files with long names")
parser.add_argument("--list", action="store_true", default=False, help="List available tests (executables)")
parser.add_argument("--list_short", action="store_true", default=False, help="List available tests (aliases)")
parser.add_argument("--list_short_main", action="store_true", default=False, help="List available tests (main repository, aliases)")
......@@ -85,11 +85,11 @@ if __name__ == "__main__":
test_args = [a for a in test_args if not a.startswith("--gtest_output=")]
if args.check:
if not [a for a in test_args if a.startswith("--perf_min_samples=")] :
if not [a for a in test_args if a.startswith("--perf_min_samples=")]:
test_args.extend(["--perf_min_samples=1"])
if not [a for a in test_args if a.startswith("--perf_force_samples=")] :
if not [a for a in test_args if a.startswith("--perf_force_samples=")]:
test_args.extend(["--perf_force_samples=1"])
if not [a for a in test_args if a.startswith("--perf_verify_sanity")] :
if not [a for a in test_args if a.startswith("--perf_verify_sanity")]:
test_args.extend(["--perf_verify_sanity"])
if bool(os.environ.get('BUILD_PRECOMMIT', None)):
......@@ -97,43 +97,42 @@ if __name__ == "__main__":
ret = 0
logs = []
for path in args.build_path:
try:
if not os.path.isdir(path):
raise Err("Not a directory (should contain CMakeCache.txt ot test executables)")
cache = CMakeCache(args.configuration)
fname = os.path.join(path, "CMakeCache.txt")
if os.path.isfile(fname):
log.debug("Reading cmake cache file: %s", fname)
cache.read(path, fname)
else:
log.debug("Assuming folder contains tests: %s", path)
cache.setDummy(path)
if args.android or cache.getOS() == "android":
log.debug("Creating Android test runner")
suite = AndroidTestSuite(args, cache, android_env)
else:
log.debug("Creating native test runner")
suite = TestSuite(args, cache)
if args.list or args.list_short or args.list_short_main:
suite.listTests(args.list_short or args.list_short_main, args.list_short_main)
else:
log.debug("Running tests in '%s', working dir: '%s'", path, args.cwd)
def parseTests(s):
return [o.strip() for o in s.split(",") if o]
l, r = suite.runTests(parseTests(args.tests), parseTests(args.blacklist), args.cwd, test_args)
logs.extend(l)
if r != 0:
ret = r
except Err as e:
log.error("ERROR: test path '%s' ==> %s", path, e.msg)
ret = -1
stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
path = args.build_path
try:
if not os.path.isdir(path):
raise Err("Not a directory (should contain CMakeCache.txt ot test executables)")
cache = CMakeCache(args.configuration)
fname = os.path.join(path, "CMakeCache.txt")
if os.path.isfile(fname):
log.debug("Reading cmake cache file: %s", fname)
cache.read(path, fname)
else:
log.debug("Assuming folder contains tests: %s", path)
cache.setDummy(path)
if args.android or cache.getOS() == "android":
log.debug("Creating Android test runner")
suite = AndroidTestSuite(args, cache, stamp, android_env)
else:
log.debug("Creating native test runner")
suite = TestSuite(args, cache, stamp)
if args.list or args.list_short or args.list_short_main:
suite.listTests(args.list_short or args.list_short_main, args.list_short_main)
else:
log.debug("Running tests in '%s', working dir: '%s'", path, args.cwd)
def parseTests(s):
return [o.strip() for o in s.split(",") if o]
logs, ret = suite.runTests(parseTests(args.tests), parseTests(args.blacklist), args.cwd, test_args)
except Err as e:
log.error("ERROR: test path '%s' ==> %s", path, e.msg)
ret = -1
if logs:
log.warning("Collected: %s", ", ".join(logs))
log.warning("Collected: %s", logs)
if ret != 0:
log.error("ERROR: some tests have failed")
......
modules/ts/misc/run_android.py
View file @ 8ba88892
#!/usr/bin/env python
import sys
from run_utils import *
import os
import re
import getpass
from run_utils import Err, log, execute, isColorEnabled, hostos
from run_suite import TestSuite
def exe(program):
return program + ".exe" if hostos == 'nt' else program
class ApkInfo:
def __init__(self):
self.pkg_name = None
......@@ -20,20 +23,17 @@ class ApkInfo:
else:
self.pkg_target = package
#==============================================================================
class Tool:
def __init__(self):
self.cmd = []
def run(self, args = [], silent = False):
def run(self, args=[], silent=False):
cmd = self.cmd[:]
cmd.extend(args)
return execute(self.cmd + args, silent)
#==============================================================================
class Adb(Tool):
def __init__(self, sdk_dir):
Tool.__init__(self)
......@@ -41,12 +41,9 @@ class Adb(Tool):
if not os.path.isfile(exe_path) or not os.access(exe_path, os.X_OK):
exe_path = None
# fix adb tool location
if not exe_path:
exe_path = getRunningProcessExePathByName("adb")
if not exe_path:
exe_path = "adb"
self.cmd = [exe_path]
self.cpuinfo = ""
def init(self, serial):
# remember current device serial. Needed if another device is connected while this script runs
......@@ -54,13 +51,9 @@ class Adb(Tool):
serial = self.detectSerial()
if serial:
self.cmd.extend(["-s", serial])
# read device cpuinfo
self.cpuinfo = self.run(["shell", "cat /proc/cpuinfo"], silent = True)
if not self.cpuinfo:
raise Err("Can not get cpuinfo from Android device")
def detectSerial(self):
adb_res = self.run(["devices"], silent = True)
adb_res = self.run(["devices"], silent=True)
# assume here that device name may consists of any characters except newline
connected_devices = re.findall(r"^[^\n]+[ \t]+device\r?$", adb_res, re.MULTILINE)
if not connected_devices:
......@@ -71,29 +64,15 @@ class Adb(Tool):
return connected_devices[0].split("\t")[0]
def getOSIdentifier(self):
return "Android" + self.run(["shell", "getprop ro.build.version.release"], silent = True).strip()
return "Android" + self.run(["shell", "getprop ro.build.version.release"], silent=True).strip()
def getHardware(self):
hw = re.search(r"^Hardware[ \t]*:[ \t]*(.*?)$", self.cpuinfo, re.MULTILINE)
if hw:
return hw.group(1).strip()
def checkArmHardware(self, expected_abi):
if expected_abi and "armeabi-v7a" in expected_abi:
if "ARMv7" not in self.cpuinfo:
raise Err("Android device does not support ARMv7 commands, but tests are built for armeabi-v7a")
if "NEON" in expected_abi and "neon" not in self.cpuinfo:
raise Err("Android device has no NEON, but tests are built for %s", expected_abi)
#==============================================================================
class Aapt(Tool):
def __init__(self, sdk_dir):
Tool.__init__(self)
aapt_fn = exe("aapt")
aapt = None
for r, ds, fs in os.walk( os.path.join(sdk_dir, 'build-tools') ):
for r, ds, fs in os.walk(os.path.join(sdk_dir, 'build-tools')):
if aapt_fn in fs:
aapt = os.path.join(r, aapt_fn)
break
......@@ -103,7 +82,7 @@ class Aapt(Tool):
def dump(self, exe):
res = ApkInfo()
output = self.run(["dump", "xmltree", exe, "AndroidManifest.xml"], silent = True)
output = self.run(["dump", "xmltree", exe, "AndroidManifest.xml"], silent=True)
if not output:
raise Err("Can not dump manifest from %s", exe)
tags = re.split(r"[ ]+E: ", output)
......@@ -111,22 +90,21 @@ class Aapt(Tool):
manifest_tag = [t for t in tags if t.startswith("manifest ")]
if not manifest_tag:
raise Err("Can not read package name from: %s", exe)
res.pkg_name = re.search(r"^[ ]+A: package=\"(?P<pkg>.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", manifest_tag[0], flags=re.MULTILINE).group("pkg")
res.pkg_name = re.search(r"^[ ]+A: package=\"(?P<pkg>.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", manifest_tag[0], flags=re.MULTILINE).group("pkg")
# get test instrumentation info
instrumentation_tag = [t for t in tags if t.startswith("instrumentation ")]
if not instrumentation_tag:
raise Err("Can not find instrumentation detials in: %s", exe)
res.pkg_runner = re.search(r"^[ ]+A: android:name\(0x[0-9a-f]{8}\)=\"(?P<runner>.*?)\" \(Raw: \"(?P=runner)\"\)\r?$", instrumentation_tag[0], flags=re.MULTILINE).group("runner")
res.pkg_target = re.search(r"^[ ]+A: android:targetPackage\(0x[0-9a-f]{8}\)=\"(?P<pkg>.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", instrumentation_tag[0], flags=re.MULTILINE).group("pkg")
res.pkg_target = re.search(r"^[ ]+A: android:targetPackage\(0x[0-9a-f]{8}\)=\"(?P<pkg>.*?)\" \(Raw: \"(?P=pkg)\"\)\r?$", instrumentation_tag[0], flags=re.MULTILINE).group("pkg")
if not res.pkg_name or not res.pkg_runner or not res.pkg_target:
raise Err("Can not find instrumentation detials in: %s", exe)
return res
#===================================================================================================
class AndroidTestSuite(TestSuite):
def __init__(self, options, cache, android_env = {}):
TestSuite.__init__(self, options, cache)
def __init__(self, options, cache, id, android_env={}):
TestSuite.__init__(self, options, cache, id)
sdk_dir = options.android_sdk or os.environ.get("ANDROID_SDK", False) or os.path.dirname(os.path.dirname(self.cache.android_executable))
log.debug("Detecting Android tools in directory: %s", sdk_dir)
self.adb = Adb(sdk_dir)
......@@ -142,14 +120,10 @@ class AndroidTestSuite(TestSuite):
def getOS(self):
return self.adb.getOSIdentifier()
def getHardware(self):
return [self.adb.getHardware()]
def checkPrerequisites(self):
self.adb.init(self.options.serial)
self.adb.checkArmHardware(self.cache.android_abi)
def runTest(self, path, logfile, workingDir, args = []):
def runTest(self, path, logfile, workingDir, args=[]):
args = args[:]
exe = os.path.abspath(path)
......@@ -160,7 +134,7 @@ class AndroidTestSuite(TestSuite):
info.forcePackage(self.options.package)
self.adb.run(["uninstall", info.pkg_name])
output = self.adb.run(["install", exe], silent = True)
output = self.adb.run(["install", exe], silent=True)
if not (output and "Success" in output):
raise Err("Can not install package: %s", exe)
......@@ -168,7 +142,7 @@ class AndroidTestSuite(TestSuite):
ret = self.adb.run(["shell", "am instrument -w %s %s/%s" % (" ".join(params), info.pkg_name, info.pkg_runner)])
return None, ret
else:
device_dir = getpass.getuser().replace(" ","") + "_" + self.options.mode +"/"
device_dir = getpass.getuser().replace(" ", "") + "_" + self.options.mode + "/"
if isColorEnabled(args):
args.append("--gtest_color=yes")
tempdir = "/data/local/tmp/"
......@@ -177,7 +151,7 @@ class AndroidTestSuite(TestSuite):
android_exe = android_dir + exename
self.adb.run(["push", exe, android_exe])
self.adb.run(["shell", "chmod 777 " + android_exe])
env_pieces = ["export %s=%s" % (a,b) for a,b in self.env.items()]
env_pieces = ["export %s=%s" % (a, b) for a, b in self.env.items()]
pieces = ["cd %s" % android_dir, "./%s %s" % (exename, " ".join(args))]
log.warning("Run: %s" % " && ".join(pieces))
ret = self.adb.run(["shell", " && ".join(env_pieces + pieces)])
......@@ -186,12 +160,11 @@ class AndroidTestSuite(TestSuite):
self.adb.run(["pull", android_dir + logfile, hostlogpath])
# cleanup
self.adb.run(["shell", "rm " + android_dir + logfile])
self.adb.run(["shell", "rm " + tempdir + "__opencv_temp.*"], silent = True)
self.adb.run(["shell", "rm " + tempdir + "__opencv_temp.*"], silent=True)
if os.path.isfile(hostlogpath):
return hostlogpath, ret
return None, ret
#===================================================================================================
if __name__ == "__main__":
log.error("This is utility file, please execute run.py script")
modules/ts/misc/run_long.py
View file @ 8ba88892
#!/usr/bin/env python
from __future__ import print_function
import xml.etree.ElementTree as ET
from glob import glob
......@@ -7,14 +6,14 @@ from pprint import PrettyPrinter as PP
LONG_TESTS_DEBUG_VALGRIND = [
('calib3d', 'Calib3d_InitUndistortRectifyMap.accuracy', 2017.22),
('dnn', 'Reproducibility*', 1000), # large DNN models
('dnn', 'Reproducibility*', 1000), # large DNN models
('features2d', 'Features2d/DescriptorImage.no_crash/3', 1000),
('features2d', 'Features2d/DescriptorImage.no_crash/4', 1000),
('features2d', 'Features2d/DescriptorImage.no_crash/5', 1000),
('features2d', 'Features2d/DescriptorImage.no_crash/6', 1000),
('features2d', 'Features2d/DescriptorImage.no_crash/7', 1000),
('imgcodecs', 'Imgcodecs_Png.write_big', 1000), # memory limit
('imgcodecs', 'Imgcodecs_Tiff.decode_tile16384x16384', 1000), # memory limit
('imgcodecs', 'Imgcodecs_Png.write_big', 1000), # memory limit
('imgcodecs', 'Imgcodecs_Tiff.decode_tile16384x16384', 1000), # memory limit
('ml', 'ML_RTrees.regression', 1423.47),
('optflow', 'DenseOpticalFlow_DeepFlow.ReferenceAccuracy', 1360.95),
('optflow', 'DenseOpticalFlow_DeepFlow_perf.perf/0', 1881.59),
......@@ -51,7 +50,7 @@ LONG_TESTS_DEBUG_VALGRIND = [
]
def longTestFilter(data, module = None):
def longTestFilter(data, module=None):
res = ['*', '-'] + [v for _, v, m in data if module is None or m == module]
return '--gtest_filter={}'.format(':'.join(res))
......
modules/ts/misc/run_suite.py
View file @ 8ba88892
#!/usr/bin/env python
import datetime
from run_utils import *
import os
import re
from run_utils import Err, log, execute, getPlatformVersion, isColorEnabled, TempEnvDir
from run_long import LONG_TESTS_DEBUG_VALGRIND, longTestFilter
timestamp = datetime.datetime.now()
class TestSuite(object):
def __init__(self, options, cache):
def __init__(self, options, cache, id):
self.options = options
self.cache = cache
self.nameprefix = "opencv_" + self.options.mode + "_"
self.tests = self.cache.gatherTests(self.nameprefix + "*", self.isTest)
self.id = id
def getOS(self):
return getPlatformVersion() or self.cache.getOS()
def getHardware(self):
res = []
if self.cache.getArch() in ["x86", "x64"] and self.cache.withCuda():
res.append("CUDA")
return res
def getLogBaseName(self, app):
global timestamp
app = self.getAlias(app)
rev = self.cache.getGitVersion()
if isinstance(timestamp, datetime.datetime):
timestamp = timestamp.strftime("%Y%m%d-%H%M%S")
if self.options.longname:
small_pieces = [self.getOS(), self.cache.getArch()] + self.cache.getDependencies() + self.getHardware() + [self.cache.getSIMDFeatures()]
big_pieces = [app, str(rev), timestamp, "_".join([p for p in small_pieces if p])]
l = "__".join(big_pieces)
else:
pieces = [app, self.cache.getOS(), self.cache.getArch()] + self.getHardware() + [rev, timestamp]
lname = "_".join([p for p in pieces if p])
lname = re.sub(r'[\(\)\[\]\s,]', '_', lname)
l = re.sub(r'_+', '_', lname)
return l
def getLogName(self, app):
return self.getLogBaseName(app) + '.xml'
return self.getAlias(app) + '_' + str(self.id) + '.xml'
def listTests(self, short = False, main = False):
def listTests(self, short=False, main=False):
if len(self.tests) == 0:
raise Err("No tests found")
for t in self.tests:
......@@ -52,7 +29,7 @@ class TestSuite(object):
log.info("%s", t)
def getAlias(self, fname):
return sorted(self.getAliases(fname), key = len)[0]
return sorted(self.getAliases(fname), key=len)[0]
def getAliases(self, fname):
def getCuts(fname, prefix):
......@@ -70,11 +47,11 @@ class TestSuite(object):
# input is full path ('/home/.../bin/opencv_test_core') or 'java'
res = [fname]
fname = os.path.basename(fname)
res.append(fname) # filename (opencv_test_core.exe)
res.append(fname) # filename (opencv_test_core.exe)
for s in getCuts(fname, self.nameprefix):
res.append(s)
if self.cache.build_type == "Debug" and "Visual Studio" in self.cache.cmake_generator:
res.append(re.sub(r"d$", '', s)) # MSVC debug config, remove 'd' suffix
res.append(re.sub(r"d$", '', s)) # MSVC debug config, remove 'd' suffix
log.debug("Aliases: %s", set(res))
return set(res)
......@@ -100,7 +77,7 @@ class TestSuite(object):
return False
return os.access(fullpath, os.X_OK)
def wrapInValgrind(self, cmd = []):
def wrapInValgrind(self, cmd=[]):
if self.options.valgrind:
res = ['valgrind']
supp = self.options.valgrind_supp or []
......@@ -114,36 +91,36 @@ class TestSuite(object):
return res + cmd + ([longTestFilter(LONG_TESTS_DEBUG_VALGRIND)] if not has_gtest_filter else [])
return cmd
def tryCommand(self, cmd):
def tryCommand(self, cmd, workingDir):
try:
if 0 == execute(cmd, cwd = workingDir):
if 0 == execute(cmd, cwd=workingDir):
return True
except:
pass
return False
def runTest(self, path, logfile, workingDir, args = []):
def runTest(self, path, logfile, workingDir, args=[]):
args = args[:]
exe = os.path.abspath(path)
if path == "java":
cmd = [self.cache.ant_executable, "-Dopencv.build.type=%s" % self.cache.build_type, "buildAndTest"]
ret = execute(cmd, cwd = self.cache.java_test_dir)
ret = execute(cmd, cwd=self.cache.java_test_dir)
return None, ret
elif path in ['python2', 'python3']:
executable = os.getenv('OPENCV_PYTHON_BINARY', None)
if executable is None:
executable = path
if not self.tryCommand([executable, '--version']):
if not self.tryCommand([executable, '--version'], workingDir):
executable = 'python'
cmd = [executable, self.cache.opencv_home + '/modules/python/test/test.py', '--repo', self.cache.opencv_home, '-v'] + args
module_suffix = '' if not 'Visual Studio' in self.cache.cmake_generator else '/' + self.cache.build_type
module_suffix = '' if 'Visual Studio' not in self.cache.cmake_generator else '/' + self.cache.build_type
env = {}
env['PYTHONPATH'] = self.cache.opencv_build + '/lib' + module_suffix + os.pathsep + os.getenv('PYTHONPATH', '')
if self.cache.getOS() == 'nt':
env['PATH'] = self.cache.opencv_build + '/bin' + module_suffix + os.pathsep + os.getenv('PATH', '')
else:
env['LD_LIBRARY_PATH'] = self.cache.opencv_build + '/bin' + os.pathsep + os.getenv('LD_LIBRARY_PATH', '')
ret = execute(cmd, cwd = workingDir, env = env)
ret = execute(cmd, cwd=workingDir, env=env)
return None, ret
else:
if isColorEnabled(args):
......@@ -157,7 +134,7 @@ class TestSuite(object):
tempDir = TempEnvDir('OPENCV_TEMP_PATH', "__opencv_temp.")
tempDir.init()
log.warning("Run: %s" % " ".join(cmd))
ret = execute(cmd, cwd = workingDir, env=env)
ret = execute(cmd, cwd=workingDir, env=env)
try:
if not self.options.valgrind and self.options.trace and int(self.options.trace_dump) >= 0:
import trace_profiler
......@@ -174,12 +151,7 @@ class TestSuite(object):
return hostlogpath, ret
return None, ret
def checkPrerequisites(self):
if self.cache.getArch() == "x64" and hostmachine == "x86":
raise Err("Target architecture is incompatible with current platform")
def runTests(self, tests, black, workingDir, args = []):
self.checkPrerequisites()
def runTests(self, tests, black, workingDir, args=[]):
args = args[:]
logs = []
test_list = self.getTestList(tests, black)
......@@ -213,7 +185,6 @@ class TestSuite(object):
logs.append(os.path.relpath(logfile, workingDir))
return logs, ret
#===================================================================================================
if __name__ == "__main__":
log.error("This is utility file, please execute run.py script")
modules/ts/misc/run_utils.py
View file @ 8ba88892
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment