Memory tool wrapper script for libyuv

This is a renamed copy of the r3354 wrapper script existing for WebRTC located
in trunk/tools/valgrind-webrtc/ of WebRTC (with suppressions cleaned).

Using the libyuv_tests.[sh,bat] script, it is possible to run the libyuv
unit tests under memory tools like Valgrind memcheck, Thread Sanitizer and
Address Sanitizer.

Adding this directory to libyuv makes it possible to handle suppressions
entirely in libyuv, instead of depending on manually setting up the WebRTC
wrapper script on the buildbots.
Having this directory in libyuv is identical to the current buildbot setup in
terms of execution, but will make the buildbot code cleaner.

It also makes it convenient for libyuv developers to run memory tests.

Examples:
memcheck: tools/valgrind-libyuv/libyuv_tests.sh -t
out/Debug/libyuv_unittest
tsan: tools/valgrind-libyuv/libyuv_tests.sh --tool=tsan -t
out/Debug/libyuv_unittest
asan: tools/valgrind-libyuv/libyuv_tests.sh --tool=asan -t
out/Debug/libyuv_unittest

BUG=none
TEST=Ran the command lines listed above.
Review URL: https://webrtc-codereview.appspot.com/1023009

git-svn-id: http://libyuv.googlecode.com/svn/trunk@533 16f28f9a-4ce2-e073-06de-1de4eb20be90

tools/valgrind-libyuv/libyuv_tests.bat

+@echo off
+:: Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
+::
+:: Use of this source code is governed by a BSD-style license
+:: that can be found in the LICENSE file in the root of the source
+:: tree. An additional intellectual property rights grant can be found
+:: in the file PATENTS.  All contributing project authors may
+:: be found in the AUTHORS file in the root of the source tree.
+
+:: This script is a copy of chrome_tests.bat with the following changes:
+:: - Invokes libyuv_tests.py instead of chrome_tests.py
+:: - Chromium's Valgrind scripts directory is added to the PYTHONPATH to make
+::   it possible to execute the Python scripts properly.
+
+:: TODO(timurrrr): batch files 'export' all the variables to the parent shell
+set THISDIR=%~dp0
+set TOOL_NAME="unknown"
+
+:: Get the tool name and put it into TOOL_NAME {{{1
+:: NB: SHIFT command doesn't modify %*
+:PARSE_ARGS_LOOP
+  if %1 == () GOTO:TOOLNAME_NOT_FOUND
+  if %1 == --tool GOTO:TOOLNAME_FOUND
+  SHIFT
+  goto :PARSE_ARGS_LOOP
+
+:TOOLNAME_NOT_FOUND
+echo "Please specify a tool (tsan or drmemory) by using --tool flag"
+exit /B 1
+
+:TOOLNAME_FOUND
+SHIFT
+set TOOL_NAME=%1
+:: }}}
+if "%TOOL_NAME%" == "drmemory"          GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "drmemory_light"    GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "drmemory_full"     GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "drmemory_pattern"  GOTO :SETUP_DRMEMORY
+if "%TOOL_NAME%" == "tsan"     GOTO :SETUP_TSAN
+echo "Unknown tool: `%TOOL_NAME%`! Only tsan and drmemory are supported."
+exit /B 1
+
+:SETUP_DRMEMORY
+if NOT "%DRMEMORY_COMMAND%"=="" GOTO :RUN_TESTS
+:: Set up DRMEMORY_COMMAND to invoke Dr. Memory {{{1
+set DRMEMORY_PATH=%THISDIR%..\..\third_party\drmemory
+set DRMEMORY_SFX=%DRMEMORY_PATH%\drmemory-windows-sfx.exe
+if EXIST %DRMEMORY_SFX% GOTO DRMEMORY_BINARY_OK
+echo "Can't find Dr. Memory executables."
+echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory"
+echo "for the instructions on how to get them."
+exit /B 1
+
+:DRMEMORY_BINARY_OK
+%DRMEMORY_SFX% -o%DRMEMORY_PATH%\unpacked -y
+set DRMEMORY_COMMAND=%DRMEMORY_PATH%\unpacked\bin\drmemory.exe
+:: }}}
+goto :RUN_TESTS
+
+:SETUP_TSAN
+:: Set up PIN_COMMAND to invoke TSan {{{1
+set TSAN_PATH=%THISDIR%..\..\third_party\tsan
+set TSAN_SFX=%TSAN_PATH%\tsan-x86-windows-sfx.exe
+if EXIST %TSAN_SFX% GOTO TSAN_BINARY_OK
+echo "Can't find ThreadSanitizer executables."
+echo "See http://www.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/threadsanitizer-on-windows"
+echo "for the instructions on how to get them."
+exit /B 1
+
+:TSAN_BINARY_OK
+%TSAN_SFX% -o%TSAN_PATH%\unpacked -y
+set PIN_COMMAND=%TSAN_PATH%\unpacked\tsan-x86-windows\tsan.bat
+:: }}}
+goto :RUN_TESTS
+
+:RUN_TESTS
+set PYTHONPATH=%THISDIR%..\python\google;%THISDIR%..\valgrind
+set RUNNING_ON_VALGRIND=yes
+python %THISDIR%libyuv_tests.py %*

tools/valgrind-libyuv/libyuv_tests.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Runs various libyuv tests through valgrind_test.py.
+
+This script inherits the chrome_tests.py in Chrome, but allows running any test
+instead of only the hard-coded ones. It uses the -t cmdline flag to do this, and
+only supports specifying a single test for each run.
+
+Suppression files:
+The Chrome valgrind directory we use as a DEPS dependency contains the following
+suppression files:
+  valgrind/memcheck/suppressions.txt
+  valgrind/memcheck/suppressions_mac.txt
+  valgrind/tsan/suppressions.txt
+  valgrind/tsan/suppressions_mac.txt
+  valgrind/tsan/suppressions_win32.txt
+Since they're referenced from the chrome_tests.py script, we have similar files
+below the directory of this script. When executing, this script will setup both
+Chrome's suppression files and our own, so we can easily maintain libyuv
+specific suppressions in our own files.
+"""
+
+import logging
+import optparse
+import os
+import sys
+
+import logging_utils
+import path_utils
+
+import chrome_tests
+
+
+class LibyuvTest(chrome_tests.ChromeTests):
+  """Class that handles setup of suppressions for libyuv.
+
+  Everything else is inherited from chrome_tests.ChromeTests.
+  """
+
+  def _DefaultCommand(self, tool, exe=None, valgrind_test_args=None):
+    """Override command-building method so we can add more suppressions."""
+    cmd = chrome_tests.ChromeTests._DefaultCommand(self, tool, exe,
+                                                   valgrind_test_args)
+    # When ChromeTests._DefaultCommand has executed, it has setup suppression
+    # files based on what's found in the memcheck/ or tsan/ subdirectories of
+    # this script's location. If Mac or Windows is executing, additional
+    # platform specific files have also been added.
+    # Since only the ones located below this directory is added, we must also
+    # add the ones maintained by Chrome, located in ../valgrind.
+
+    # The idea is to look for --suppression arguments in the cmd list and add a
+    # modified copy of each suppression file, for the corresponding file in
+    # ../valgrind. If we would simply replace 'valgrind-libyuv' with 'valgrind'
+    # we may produce invalid paths if other parts of the path contain that
+    # string. That's why the code below only replaces the end of the path.
+    script_dir = path_utils.ScriptDir()
+    old_base, _ = os.path.split(script_dir)
+    new_dir = os.path.join(old_base, 'valgrind')
+    add_suppressions = []
+    for token in cmd:
+      if '--suppressions' in token:
+        add_suppressions.append(token.replace(script_dir, new_dir))
+    return add_suppressions + cmd
+
+
+def main(_):
+  parser = optparse.OptionParser('usage: %prog -b <dir> -t <test> <test args>')
+  parser.disable_interspersed_args()
+  parser.add_option('-b', '--build_dir',
+                    help=('Location of the compiler output. Can only be used '
+                          'when the test argument does not contain this path.'))
+  parser.add_option('-t', '--test', help='Test to run.')
+  parser.add_option('', '--baseline', action='store_true', default=False,
+                    help='Generate baseline data instead of validating')
+  parser.add_option('', '--gtest_filter',
+                    help='Additional arguments to --gtest_filter')
+  parser.add_option('', '--gtest_repeat',
+                    help='Argument for --gtest_repeat')
+  parser.add_option('-v', '--verbose', action='store_true', default=False,
+                    help='Verbose output - enable debug log messages')
+  parser.add_option('', '--tool', dest='valgrind_tool', default='memcheck',
+                    help='Specify a valgrind tool to run the tests under')
+  parser.add_option('', '--tool_flags', dest='valgrind_tool_flags', default='',
+                    help='Specify custom flags for the selected valgrind tool')
+  parser.add_option('', '--keep_logs', action='store_true', default=False,
+                    help=('Store memory tool logs in the <tool>.logs directory '
+                          'instead of /tmp.\nThis can be useful for tool '
+                          'developers/maintainers.\nPlease note that the <tool>'
+                          '.logs directory will be clobbered on tool startup.'))
+  options, args = parser.parse_args()
+
+  if options.verbose:
+    logging_utils.config_root(logging.DEBUG)
+  else:
+    logging_utils.config_root()
+
+  if not options.test:
+    parser.error('--test not specified')
+
+  # If --build_dir is provided, prepend it to the test executable if needed.
+  test_executable = options.test
+  if options.build_dir and not test_executable.startswith(options.build_dir):
+    test_executable = os.path.join(options.build_dir, test_executable)
+  args = [test_executable] + args
+
+  test = LibyuvTest(options, args, 'cmdline')
+  return test.Run()
+
+if __name__ == '__main__':
+  return_code = main(sys.argv)
+  sys.exit(return_code)

tools/valgrind-libyuv/libyuv_tests.sh

+#!/bin/bash
+# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+# Set up some paths and re-direct the arguments to libyuv_tests.py
+
+# This script is a copy of the chrome_tests.sh wrapper script with the following
+# changes:
+# - The locate_valgrind.sh of Chromium's Valgrind scripts dir is used to locate
+#   the Valgrind framework install.
+# - libyuv_tests.py is invoked instead of chrome_tests.py.
+# - Chromium's Valgrind scripts directory is added to the PYTHONPATH to make it
+#   possible to execute the Python scripts properly.
+
+export THISDIR=`dirname $0`
+ARGV_COPY="$@"
+
+# We need to set CHROME_VALGRIND iff using Memcheck or TSan-Valgrind:
+#   tools/valgrind-libyuv/libyuv_tests.sh --tool memcheck
+# or
+#   tools/valgrind-libyuv/libyuv_tests.sh --tool=memcheck
+# (same for "--tool=tsan")
+tool="memcheck"  # Default to memcheck.
+while (( "$#" ))
+do
+  if [[ "$1" == "--tool" ]]
+  then
+    tool="$2"
+    shift
+  elif [[ "$1" =~ --tool=(.*) ]]
+  then
+    tool="${BASH_REMATCH[1]}"
+  fi
+  shift
+done
+
+NEEDS_VALGRIND=0
+NEEDS_DRMEMORY=0
+
+case "$tool" in
+  "memcheck")
+    NEEDS_VALGRIND=1
+    ;;
+  "tsan" | "tsan_rv")
+    if [ "`uname -s`" == CYGWIN* ]
+    then
+      NEEDS_PIN=1
+    else
+      NEEDS_VALGRIND=1
+    fi
+    ;;
+  "drmemory" | "drmemory_light" | "drmemory_full" | "drmemory_pattern")
+    NEEDS_DRMEMORY=1
+    ;;
+esac
+
+# For Libyuv, we'll use the locate_valgrind.sh script in Chromium's Valgrind
+# scripts dir to locate the Valgrind framework install
+CHROME_VALGRIND_SCRIPTS=$THISDIR/../valgrind
+
+if [ "$NEEDS_VALGRIND" == "1" ]
+then
+  CHROME_VALGRIND=`sh $CHROME_VALGRIND_SCRIPTS/locate_valgrind.sh`
+  if [ "$CHROME_VALGRIND" = "" ]
+  then
+    # locate_valgrind.sh failed
+    exit 1
+  fi
+  echo "Using valgrind binaries from ${CHROME_VALGRIND}"
+
+  PATH="${CHROME_VALGRIND}/bin:$PATH"
+  # We need to set these variables to override default lib paths hard-coded into
+  # Valgrind binary.
+  export VALGRIND_LIB="$CHROME_VALGRIND/lib/valgrind"
+  export VALGRIND_LIB_INNER="$CHROME_VALGRIND/lib/valgrind"
+
+  # Clean up some /tmp directories that might be stale due to interrupted
+  # chrome_tests.py execution.
+  # FYI:
+  #   -mtime +1  <- only print files modified more than 24h ago,
+  #   -print0/-0 are needed to handle possible newlines in the filenames.
+  echo "Cleanup /tmp from Valgrind stuff"
+  find /tmp -maxdepth 1 \(\
+        -name "vgdb-pipe-*" -or -name "vg_logs_*" -or -name "valgrind.*" \
+      \) -mtime +1 -print0 | xargs -0 rm -rf
+fi
+
+if [ "$NEEDS_DRMEMORY" == "1" ]
+then
+  if [ -z "$DRMEMORY_COMMAND" ]
+  then
+    DRMEMORY_PATH="$THISDIR/../../third_party/drmemory"
+    DRMEMORY_SFX="$DRMEMORY_PATH/drmemory-windows-sfx.exe"
+    if [ ! -f "$DRMEMORY_SFX" ]
+    then
+      echo "Can't find Dr. Memory executables."
+      echo "See http://www.chromium.org/developers/how-tos/using-valgrind/dr-memory"
+      echo "for the instructions on how to get them."
+      exit 1
+    fi
+
+    chmod +x "$DRMEMORY_SFX"  # Cygwin won't run it without +x.
+    "$DRMEMORY_SFX" -o"$DRMEMORY_PATH/unpacked" -y
+    export DRMEMORY_COMMAND="$DRMEMORY_PATH/unpacked/bin/drmemory.exe"
+  fi
+fi
+
+if [ "$NEEDS_PIN" == "1" ]
+then
+  if [ -z "$PIN_COMMAND" ]
+  then
+    # Set up PIN_COMMAND to invoke TSan.
+    TSAN_PATH="$THISDIR/../../third_party/tsan"
+    TSAN_SFX="$TSAN_PATH/tsan-x86-windows-sfx.exe"
+    echo "$TSAN_SFX"
+    if [ ! -f $TSAN_SFX ]
+    then
+      echo "Can't find ThreadSanitizer executables."
+      echo "See http://www.chromium.org/developers/how-tos/using-valgrind/threadsanitizer/threadsanitizer-on-windows"
+      echo "for the instructions on how to get them."
+      exit 1
+    fi
+
+    chmod +x "$TSAN_SFX"  # Cygwin won't run it without +x.
+    "$TSAN_SFX" -o"$TSAN_PATH"/unpacked -y
+    export PIN_COMMAND="$TSAN_PATH/unpacked/tsan-x86-windows/tsan.bat"
+  fi
+fi
+
+# Add Chrome's Valgrind scripts dir to the PYTHON_PATH since it contains
+# the scripts that are needed for this script to run
+PYTHONPATH=$THISDIR/../python/google:$CHROME_VALGRIND_SCRIPTS python \
+           "$THISDIR/libyuv_tests.py" $ARGV_COPY

tools/valgrind-libyuv/memcheck/OWNERS

+*

tools/valgrind-libyuv/memcheck/PRESUBMIT.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""
+Copied from Chrome's src/tools/valgrind/memcheck/PRESUBMIT.py
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
+for more details on the presubmit API built into gcl.
+"""
+
+import os
+import re
+import sys
+
+def CheckChange(input_api, output_api):
+  """Checks the memcheck suppressions files for bad data."""
+
+  # Add the path to the Chrome valgrind dir to the import path:
+  tools_vg_path = os.path.join(input_api.PresubmitLocalPath(), '..', '..',
+                               'valgrind')
+  sys.path.append(tools_vg_path)
+  import suppressions
+
+  sup_regex = re.compile('suppressions.*\.txt$')
+  suppressions = {}
+  errors = []
+  check_for_memcheck = False
+  # skip_next_line has 3 possible values:
+  # - False: don't skip the next line.
+  # - 'skip_suppression_name': the next line is a suppression name, skip.
+  # - 'skip_param': the next line is a system call parameter error, skip.
+  skip_next_line = False
+  for f in filter(lambda x: sup_regex.search(x.LocalPath()),
+                  input_api.AffectedFiles()):
+    for line, line_num in zip(f.NewContents(),
+                              xrange(1, len(f.NewContents()) + 1)):
+      line = line.lstrip()
+      if line.startswith('#') or not line:
+        continue
+
+      if skip_next_line:
+        if skip_next_line == 'skip_suppression_name':
+          if 'insert_a_suppression_name_here' in line:
+            errors.append('"insert_a_suppression_name_here" is not a valid '
+                          'suppression name')
+          if suppressions.has_key(line):
+            if f.LocalPath() == suppressions[line][1]:
+              errors.append('suppression with name "%s" at %s line %s '
+                            'has already been defined at line %s' %
+                            (line, f.LocalPath(), line_num,
+                             suppressions[line][1]))
+            else:
+              errors.append('suppression with name "%s" at %s line %s '
+                            'has already been defined at %s line %s' %
+                            (line, f.LocalPath(), line_num,
+                             suppressions[line][0], suppressions[line][1]))
+          else:
+            suppressions[line] = (f, line_num)
+            check_for_memcheck = True;
+        skip_next_line = False
+        continue
+      if check_for_memcheck:
+        if not line.startswith('Memcheck:'):
+          errors.append('"%s" should be "Memcheck:..." in %s line %s' %
+                        (line, f.LocalPath(), line_num))
+        check_for_memcheck = False;
+      if line == '{':
+        skip_next_line = 'skip_suppression_name'
+        continue
+      if line == "Memcheck:Param":
+        skip_next_line = 'skip_param'
+        continue
+
+      if (line.startswith('fun:') or line.startswith('obj:') or
+          line.startswith('Memcheck:') or line == '}' or
+          line == '...'):
+        continue
+      errors.append('"%s" is probably wrong: %s line %s' % (line, f.LocalPath(),
+                                                            line_num))
+  if errors:
+    return [output_api.PresubmitError('\n'.join(errors))]
+  return []
+
+def CheckChangeOnUpload(input_api, output_api):
+  return CheckChange(input_api, output_api)
+
+def CheckChangeOnCommit(input_api, output_api):
+  return CheckChange(input_api, output_api)
+
+def GetPreferredTrySlaves():
+  # We don't have any memcheck slaves yet, so there's no use for this method.
+  # When we have, the slave name(s) should be put into this list.
+  return []

tools/valgrind-libyuv/memcheck/suppressions.txt

+# This file is used in addition to the one already maintained in Chrome.
+# It acts as a place holder for future additions for this project.
+# It must exist for the Python wrapper script to work properly.
+
+

tools/valgrind-libyuv/memcheck/suppressions_mac.txt

+# This file is used in addition to the one already maintained in Chrome.
+# It acts as a place holder for future additions for this project.
+# It must exist for the Python wrapper script to work properly.
+
+

tools/valgrind-libyuv/memcheck/suppressions_win32.txt

+# This file is used in addition to the one already maintained in Chrome.
+# It acts as a place holder for future additions for this project.
+# It must exist for the Python wrapper script to work properly.
+
+

tools/valgrind-libyuv/tsan/OWNERS

+*

tools/valgrind-libyuv/tsan/PRESUBMIT.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The LibYuv Project Authors. All rights reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import os
+import re
+import sys
+
+"""
+Copied from Chrome's src/tools/valgrind/tsan/PRESUBMIT.py
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
+for more details on the presubmit API built into gcl.
+"""
+
+def CheckChange(input_api, output_api):
+  """Checks the TSan suppressions files for bad suppressions."""
+
+  # Add the path to the Chrome valgrind dir to the import path:
+  tools_vg_path = os.path.join(input_api.PresubmitLocalPath(), '..', '..',
+                               'valgrind')
+  sys.path.append(tools_vg_path)
+  import suppressions
+
+  return suppressions.PresubmitCheck(input_api, output_api)
+
+def CheckChangeOnUpload(input_api, output_api):
+  return CheckChange(input_api, output_api)
+
+def CheckChangeOnCommit(input_api, output_api):
+  return CheckChange(input_api, output_api)
+
+def GetPreferredTrySlaves():
+  # We don't have any tsan slaves yet, so there's no use for this method.
+  # When we have, the slave name(s) should be put into this list.
+  return []

tools/valgrind-libyuv/tsan/suppressions.txt

+# This file is used in addition to the one already maintained in Chrome.
+# It acts as a place holder for future additions for this project.
+# It must exist for the Python wrapper script to work properly.
+
+

tools/valgrind-libyuv/tsan/suppressions_mac.txt

+# This file is used in addition to the one already maintained in Chrome.
+# It acts as a place holder for future additions for this project.
+# It must exist for the Python wrapper script to work properly.
+
+

tools/valgrind-libyuv/tsan/suppressions_win32.txt

+# This file is used in addition to the one already maintained in Chrome.
+# It acts as a place holder for future additions for this project.
+# It must exist for the Python wrapper script to work properly.
+
+