CodeTest.py

Go to the documentation of this file.

#
# Copyright 2016, MeVis Medical Solutions AG
#
# The user may use this file in accordance with the license agreement provided with
# the Software or, alternatively, in accordance with the terms contained in a
# written agreement between the user and MeVis Medical Solutions AG.
#
# For further information use the contact form at https://www.mevislab.de/contact
#
 
 
 
from TestSupport.Macros import EXPECT_EQ, EXPECT_TRUE, ASSERT_TRUE
from TestSupport.Logging import infoHTML, errorHTML
from mevis import MLAB, MLABPackageManager, MLABTestCaseDatabase
import os
import os.path
import sys
import re
import json
 
# Allows to inject test functions for a compiled GoogleTest executable into a FunctionalTestCase
# Example (to be put into a TestCase's python file:
#   from TestSupport import CodeTest
#   CodeTest.inject("$(MLAB_MeVisLab_Standard)/CodeTests/bin/MLBaseTest", ctx)
def inject(executable, context):
  # replace package variable with the path to the binary directory in this case:
  # (the default would be the source directory, which might not be the same for some setups)
  match = re.search(r"\$\‍(MLAB_([^_)]+)_([^_)]+)\‍)", executable)
  if match:
    package = MLABPackageManager.packageByIdentifier(match.group(1) + "/" + match.group(2))
    if package:
      executable = executable.replace(match.group(0), package.binariesPath())
  test = CodeTest(executable = context.expandFilename(executable))
  test.injectTestFunctions(context)
 
pathToBuildSystemScripts = MLABPackageManager.getPathInPackage('MeVis/BuildSystem', 'BuildTools/Scripts')
if os.path.isdir(pathToBuildSystemScripts):
  if not pathToBuildSystemScripts in sys.path:
    sys.path.insert(0, pathToBuildSystemScripts)
  from MeVisPython import MeVisPython
del pathToBuildSystemScripts
 
def _parseGoogleTestOutput(result, allowRestart=False):
  colorOk = 'color: rgb(78,154,6);'
  colorFailed = 'color: rgb(204,0,0);'
  commonStyle = 'font-family: monospace;'
 
  statusFormat = '<div style="%s"><span style="%s">%%s</span>%%s</div>'
  errorFormat = statusFormat % (commonStyle, colorFailed)
  infoFormat = statusFormat % (commonStyle, colorOk)
  
  statusRegExp = re.compile("(?P<status>\\[\\s*\\S+\\s*\\])(?P<message>.*)")
  
  hasNotifiedStdErrOutput = False
 
  # Check for concrete error or failure information in stderr message. If yes then 
  # really take it as an error. Often we have normal stderr prints of suppressed 
  # leaks from sanitizer or stupidly other informational but non-failure-messages, 
  # which should not be handled as error. If they do not actually print 
  # something like "error:" or "failed:" they should be tolerated.
  lowerStderrMsg = result['stderr'].lower()
  stdErrContainsError = (lowerStderrMsg.find("error") >=0) or (lowerStderrMsg.find("failed") >=0)
 
  needsRestartWithoutSanitizer = False
  if stdErrContainsError:
    # Check whether test failed with a sanitizer failure caused by a sanitizer bug. It makes a 
    # kind of unit test incompatible.
    if lowerStderrMsg.find("sanitizer check failed:") >=0 and \
       lowerStderrMsg.find("src/libsanitizer/asan/asan_malloc_linux.cc") >= 0 and \
       lowerStderrMsg.find("allocated_for_dlsym") >= 0:
      needsRestartWithoutSanitizer = allowRestart
 
  if not needsRestartWithoutSanitizer:
    
    EXPECT_EQ(result["status"], 0, msg='The test status is not 0.', logOnSuccess=False)
    EXPECT_TRUE(result["ok"], msg='The test run failed.', logOnSuccess=False)
    ASSERT_TRUE("stdout" in result, msg='The test output does not exists.', logOnSuccess=False)
    
    for outChannel in ['stdout', 'stderr']:
      for line in result[outChannel].splitlines():
        # Before first std::err message print a notice that there are also std::cerr messages.
        if outChannel=='stderr' and not hasNotifiedStdErrOutput:
          if stdErrContainsError:
            errorHTML(('Note: The stderr channel output contained "error" or "failed" which is handled as test failure.'))
          infoHTML('The test printed the following messages to stderr channel:')
          hasNotifiedStdErrOutput = True
          
        m = statusRegExp.match(line)
        if m:
          status = m.group('status').replace(' ', '&nbsp;').replace('\t', '&nbsp;'*4)
          message = m.group('message').replace(' ', '&nbsp;').replace('\t', '&nbsp;'*4)
          if 'FAILED' in status:
            errorHTML(errorFormat % (status, message))
          else:
            infoHTML(infoFormat % (status, message))
        else:
          line = line.replace(' ', '&nbsp;').replace('\t', '&nbsp;'*4)
          infoHTML(('<div style="%s">' + line + '</div>') % commonStyle)
 
  return needsRestartWithoutSanitizer
 
class TestFunctionGroup:
  def __init__(self, name, sourceName):
    self.namename = name
    self.sourceNamesourceName = sourceName
    self.__testFunctions__testFunctions = []
  def addFunction(self, function):
    self.__testFunctions__testFunctions.append(function)
  def getFunctions(self):
    return self.__testFunctions__testFunctions
 
 
class TestFunction:
  def __init__(self, name, sourceName):
    self.namename = name
    self.sourceNamesourceName = sourceName
 
 
class CodeTest:
  # Either proFile or executable need to be passed in. Executable is without debug and "
  def __init__(self, cmakeFile = None, executable = None):
    self.__testFunctionGroups__testFunctionGroups = {}
    self.__infoDict__infoDict = None
    self.__uniqueFunctionNames__uniqueFunctionNames = None
    self.cmakeFilecmakeFile = cmakeFile
    self.executableexecutable = executable
    self.__searchedExecutable__searchedExecutable = False
    
    if cmakeFile is not None:
      self.namename = self._parseCMakeFile_parseCMakeFile(cmakeFile)
    
  def _getUniqueTestCaseName(self, name):
    uniqueName = name
    i = 0
    while MLABTestCaseDatabase.testCaseInfo(uniqueName):
      if i == 0:
        # first append _CodeTest, and later also a number
        name += '_CodeTest'
        uniqueName = name
      else:
        uniqueName = name + str(i)
      i += 1
    return uniqueName
    
  def getInfoDict(self):
    if self.__infoDict__infoDict == None:
      package = MLABPackageManager.findPackageContainingPath(self.cmakeFilecmakeFile)
      self.__infoDict__infoDict = {'package': package.packageIdentifier(),
                         'author': '',
                         'file': self.cmakeFilecmakeFile,
                         'lineno': 1,
                         'type': 'CodeTestTestCase',
                         'scriptFile': 'ScriptFileNotAvailable'}
    return self.__infoDict__infoDict
    
  def getTestFunctionGroups(self): return self.__testFunctionGroups
    
  def getExecutable(self):
    if self.executableexecutable is None and not self.__searchedExecutable__searchedExecutable:
      self.__searchedExecutable__searchedExecutable = True
      self.executableexecutable = self._findExecutables_findExecutables()
      if not self.__executableExists__executableExists(self.executableexecutable):
        MLAB.logError("Could not locate CodeTest executable: " + self.namename)
    else:
      if not os.path.exists(self.executableexecutable):
        # turn base filename into something existing
        exeSuffix = ".exe" if MLAB.isWindows() else ""
        if os.path.exists(self.executableexecutable + exeSuffix):
          self.executableexecutable += exeSuffix
        # fall back to debug executable if release executable does not exist:
        elif os.path.exists(self.executableexecutable + "_d" + exeSuffix):
          self.executableexecutable += "_d" + exeSuffix
    return self.executableexecutable
  
  @staticmethod
  def __executableExists(executable):
    return executable and os.path.exists(executable)
  
  def existsExecutable(self):
    return self.__executableExists__executableExists(self.getExecutablegetExecutable())
 
  def buildProject(self):
    if self.existsExecutableexistsExecutable():
      os.remove(self.executableexecutable)
    executable = 'cmake'
    arguments = ['--build', '.', '--config', ('Debug' if MLAB.isDebug() else 'Release'), '--target', self.namename]
    result = MLAB.runCommandStdInOut(executable, arguments, self._getBuildRoot_getBuildRoot())
    # force to search for the executable again:
    self.executableexecutable = None
    self.__searchedExecutable__searchedExecutable = False
    if not self.existsExecutableexistsExecutable():
      MLAB.showCritical(self._getError_getError(result))
       
    if result['ok'] != True:
      MLAB.logError(self._getError_getError(result))
    return result['ok'] == True     
 
  def _getError(self, result):
    message = []
    stderr = result.get('stderr')
    stdout = result.get('stdout')
    if stdout: message.append(stdout.encode('latin1').decode('UTF-8'))
    if stderr: message.append(stderr.encode('latin1').decode('UTF-8'))
    if not stdout and not stderr:
      message.append('An unknown error occurred')
    return '\n'.join(message)
 
  def _parseCMakeFile(self, cmakeFile):    
    testRegex = re.compile(r'^(?!#)\s*mlab_add_test\‍(\s*(\w*)')
    target = None
    with open(cmakeFile, 'r') as f:
      for line in f:
        match = testRegex.match(line)
        if match and match.group(1):
          target = match.group(1)
          return target
    return target
  
  def _getBuildRoot(self):
    if 'MLAB_BUILD' in os.environ:
      buildRoot = os.environ['MLAB_BUILD']
    else:
      buildRoot = os.path.normpath(os.path.join(MLABPackageManager.packageByIdentifier('MeVisLab/IDE').binariesPath(), "..", ".."))
    return buildRoot
  
  def _findExecutables(self):
    executable = None
    command = 'ctest'
    testArguments = ["-V", "-R", self.namename, "-C", ("Debug" if MLAB.isDebug() else "Release"), "--show-only=json-v1"]
    result = MLAB.runCommandStdInOut(command, testArguments, self._getBuildRoot_getBuildRoot())
    if result['ok'] == True:
      testDetails = json.loads(result['stdout'])
      if 'tests' in testDetails and len(testDetails['tests'])>0 and 'command' in testDetails['tests'][0] and len(testDetails['tests'][0]['command']) > 0:
        executable = testDetails['tests'][0]['command'][0]
    return executable
  
  def loadTestFunctions(self):
    if not self.existsExecutableexistsExecutable():
      self.buildProjectbuildProject()
    if self.existsExecutableexistsExecutable():
      executable = self.executableexecutable
      result = MLAB.runCommandStdInOut(executable, ["--gtest_list_tests"], os.path.dirname(executable))
      if result['ok'] == True:
        self._parseTestFunctionsFromOutput_parseTestFunctionsFromOutput(result['stdout'])
  
  def __getUniqueFunctionName(self, name):
    i = 0
    uniqueName = name
    while uniqueName in self.__uniqueFunctionNames__uniqueFunctionNames:
      i += 1
      uniqueName = name + str(i)
    self.__uniqueFunctionNames__uniqueFunctionNames.add(uniqueName)
    return uniqueName
    
  def _parseTestFunctionsFromOutput(self, output):
    identifierRegExp = re.compile(r'^[\S]+$')
    self.__testFunctionGroups__testFunctionGroups = []
    self.__uniqueFunctionNames__uniqueFunctionNames = set()
    group = None
    groupIdx = 1
    functionIdx = 1
    for line in output.splitlines():
      commentIdx = line.find('#')
      if commentIdx >= 0:
        line = line[:commentIdx].rstrip(' ')
      if not line.startswith(' '):
        name = line.rstrip('.')
        if identifierRegExp.match(name):
          sourceName = 'GROUP%03d_%s' % (groupIdx, name.replace('/', '_'))
          group = TestFunctionGroup(name, sourceName)
          self.__testFunctionGroups__testFunctionGroups.append(group)
          groupIdx += 1
      else:
        name = line.strip()
        if identifierRegExp.match(name):
          isDisabled = False
          uniqueName = name
          if uniqueName.startswith('DISABLED_'):
            uniqueName = uniqueName[len('DISABLED_'):]
            isDisabled = True
          uniqueName = self.__getUniqueFunctionName__getUniqueFunctionName(uniqueName)
          sourceName = 'TEST%03d_%s' % (functionIdx, uniqueName.replace('/', '_'))
          if isDisabled:
            sourceName = 'DISABLED_' + sourceName
          function = TestFunction(name, sourceName)
          group.addFunction(function)
          functionIdx += 1
  
  def injectTestFunctions(self, context):
    self.loadTestFunctionsloadTestFunctions()
 
    testCode = ''
    ctx = context
    
    globalSymbols = globals()
    localSymbols= locals()
    
    for group in self.getTestFunctionGroupsgetTestFunctionGroups():
      for function in group.getFunctions():
        testCode += self._functionCode_functionCode(group, function)
      testCode += self._groupCode_groupCode(group)
 
    eval(compile(testCode, '<string>', 'exec'), globalSymbols, localSymbols)
 
  def _groupCode(self, group):
    functions = []
    for f in group.getFunctions():
      functions.append("'" + f.sourceName + "'") 
    return '''def %(sourceName)s(): 
  return %(functions)s,
ctx.setScriptVariable(%(sourceName)s.__name__, %(sourceName)s)
''' % dict(sourceName = group.sourceName, functions = ",".join(functions))
 
  def _hasSanitizerSuppression(self, sanitizerPrefix, envVar, compilerVersion):
    mlabRootPath = MLAB.variable('MLAB_MeVis_BuildSystem')
    suppressionFile = mlabRootPath + '/BuildTools/linux/' + sanitizerPrefix + u'SanitizerSuppressions_gcc' + compilerVersion + '.supp'
    #print(suppressionFile)
    return os.path.isfile(suppressionFile) and os.path.exists(suppressionFile)
    
  def _determineSanitizerSetUpCode(self):
    if MLAB.isLinux():
      compilerInfo = MLAB.compilerInfo()
      compilerVersion = '4'
      if compilerInfo.find(u'GCC_64_7') >= 0:
        compilerVersion = u'7'
      elif compilerInfo.find(u'GCC_64_9') >= 0:
        compilerVersion = u'9'
      elif compilerInfo.find(u'') >= 0:
        compilerVersion = u'5'
      
      hasSuppressions = self._hasSanitizerSuppression_hasSanitizerSuppression('Leak', 'LSAN_OPTIONS', compilerVersion) and \
                        self._hasSanitizerSuppression_hasSanitizerSuppression('Address', 'ASAN_OPTIONS', compilerVersion) and \
                        self._hasSanitizerSuppression_hasSanitizerSuppression('UB', 'UBSAN_OPTIONS', compilerVersion) and \
                        self._hasSanitizerSuppression_hasSanitizerSuppression('Thread', 'TSAN_OPTIONS', compilerVersion)
      if hasSuppressions:
        return '''
  MLAB.log(u"Running in sanitizer environment of gcc%(compilerVer)s ('asan' and 'ubsan' libs with LD_PRELOAD as well as suppression files).")
  mlabRootPath = MLAB.variable("MLAB_MeVis_BuildSystem")
  sanitizerEnvVarIntro = "suppressions=" + mlabRootPath + os.sep
  os.environ["LSAN_OPTIONS"] = sanitizerEnvVarIntro + u"BuildTools/linux/LeakSanitizerSuppressions_gcc%(compilerVer)s.supp"
  os.environ["ASAN_OPTIONS"] = sanitizerEnvVarIntro + u"BuildTools/linux/AddressSanitizerSuppressions_gcc%(compilerVer)s.supp"
  os.environ["UBSAN_OPTIONS"] = sanitizerEnvVarIntro + u"BuildTools/linux/UBSanitizerSuppressions_gcc%(compilerVer)s.supp"
  os.environ["TSAN_OPTIONS"] = sanitizerEnvVarIntro + u"linux/ThreadSanitizerSuppressions_gcc%(compilerVer)s.supp"
  os.environ["LD_PRELOAD"] = u"/usr/lib/gcc/x86_64-linux-gnu/%(compilerVer)s/libasan.so:/usr/lib/gcc/x86_64-linux-gnu/%(compilerVer)s/libubsan.so"
  ''' % dict(compilerVer = compilerVersion)
      else:
        return '''
  MLAB.log(u"Could not find (all) sanitizer suppression files: not running test in gcc sanitizer environment.")
  '''
    else:
      return ''
 
  def _functionCode(self, group, function):
    exePath = self.getExecutablegetExecutable()
    exePath = exePath.replace('\\', '\\\\')
    saniCode = self._determineSanitizerSetUpCode_determineSanitizerSetUpCode()
    return '''def %(sourceName)s():
  executable = "%(executable)s"
  for package in MLABPackageManager.packages():
    os.environ[package.packageEnvVariableName()] = package.scriptsPath()
%(sanitizerSetUpCode)s
  result = MLAB.runCommandStdInOut(executable, ['--gtest_also_run_disabled_tests', '--gtest_filter=%(groupName)s.%(functionName)s'], os.path.dirname(executable))
  from TestSupport.CodeTest import _parseGoogleTestOutput
  needsRestartWithoutSanitizer = _parseGoogleTestOutput(result, True)
  if needsRestartWithoutSanitizer:
    os.environ["LD_PRELOAD"] = ""
    MLAB.log(u"Sanitizer is incompatible with unittest. Restarting test without sanitzer preload.")
    result = MLAB.runCommandStdInOut(executable, ['--gtest_also_run_disabled_tests', '--gtest_filter=%(groupName)s.%(functionName)s'], os.path.dirname(executable))
    _parseGoogleTestOutput(result)
ctx.setScriptVariable(%(sourceName)s.__name__, %(sourceName)s)
''' % dict(sourceName = function.sourceName, executable = exePath, sanitizerSetUpCode = saniCode, groupName = group.name, functionName = function.name)