TestCase.py

Go to the documentation of this file.

#
# Copyright 2009, MeVis Medical Solutions AG
#
# The user may use this file in accordance with the license agreement provided with
# the Software or, alternatively, in accordance with the terms contained in a
# written agreement between the user and MeVis Medical Solutions AG.
#
# For further information use the contact form at https://www.mevislab.de/contact
#
 
 
 
# -- system imports ----------------------------------------------------------------------------{{{-
import inspect
import os
import re
import sys
import time
import traceback
import unittest
import asyncio
from PythonQt import QtCore
 
if sys.version_info.major >= 3:
  unicode = str
 
import xml.etree.cElementTree as etree
# ----------------------------------------------------------------------------------------------}}}-
 
# -- local imports -----------------------------------------------------------------------------{{{-
import mevis
from TestSupport.FieldValueTests import FieldValueTestCaseSet
 
from TestSupport.TestHelper import TestHelper, CancelTestException
from . import Utils
import TestSupport.Logging
import TestSupport.Macros
 
# ----------------------------------------------------------------------------------------------}}}-
 
# -- local imports -----------------------------------------------------------------------------{{{-
MLABpriv = mevis.MLAB.priv()
# ----------------------------------------------------------------------------------------------}}}-
 
# Prints the exception that occurred during a test function.
def printTestFunctionException():
  exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
  tb = exceptionTraceback.tb_next
  traceback.print_tb(tb, limit=None, file=sys.stderr)
  lines = traceback.format_exception_only(exceptionType, exceptionValue)
  for line in lines:
    sys.stderr.write(line)
 
 
class RunTestFunctionData:
  def __init__(self, funcNode, funcName, logHandler, isLastTestFunction):
    self.funcNode = funcNode
    self.funcName = funcName
    self.logHandler = logHandler
    self.isLastTestFunction = isLastTestFunction
    self.started = None
    self.cancelled = False
 
 
class UnitTestResult(unittest.TestResult):
  
  # deactivate stdout/stderr redirection, the output is captured by the TestCenter anyway
  def _setupStdout(self):
    pass
 
  def _restoreStdout(self):
    pass
  
  def logResult(self, successMsg):
    if self.wasSuccessful() and successMsg:
      mevis.MLAB.log( successMsg )
    if self.errors:
      for error in self.errors:
        mevis.MLAB.logError( error[ 1 ] )
    if self.failures:
      for failure in self.failures:
        mevis.MLAB.logError( failure[ 1 ] )
 
 
# -- class TestCase ----------------------------------------------------------------------------{{{-
 
class TestCase:
  # -- member variables ------------------------------------------------------------------------{{{-
  
  _ctx = None
  
  _testHelper = None
 
  
  _testCaseName = None
  
  _testCaseType = None
 
  
  __xmlRoot = None
  
  __functionDict = None
 
  
  __valid = None
 
  # XML node for extra test case results
  __extraTestCasesResults = None
  
  __currentResultNode = None
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __init__ ----------------------------------------------------------------------------{{{-
  
  def __init__ (self, testCaseName, isNewlyCreated=False):
    self._testCaseName = testCaseName
    self._testHelper = TestHelper.getInstance()
    self.__testCaseListener = None
 
    self.__collectedStartupLog = None
    self.__collectedStartupErrorsState = False
    self.__collectedStartupWarningsState = False
    self.__setupTestCaseCalled = False
 
    self.__valid = False
    self.__functionDict = {}
 
    
    self.__dummyTestSuite = None
    
    
    self.__previousTestClass = None
 
    infoDict = self._getTestCaseInfo(testCaseName)
 
    # Get the basic test case information and store them in the XML node.
    self.__xmlRoot = etree.Element("TestCase", name=self._testCaseName, type=self._testCaseType)
    self.__xmlRoot.set('timeout', infoDict.get('timeout', '0'))
    self.__perFunctionResultDirectory = mevis.MLAB.valueIsTrue(infoDict.get('perFunctionResultDirectory', '1'))
 
    etree.SubElement(self.__xmlRoot, 'Package').text = infoDict['package']
    etree.SubElement(self.__xmlRoot, 'Author').text = infoDict['author']
    etree.SubElement(self.__xmlRoot, 'Maintainer').text = infoDict.get('maintainer', '')
    etree.SubElement(self.__xmlRoot, 'Comment').text = infoDict.get('comment', '')
    etree.SubElement(self.__xmlRoot, 'File').text = infoDict['file']
    etree.SubElement(self.__xmlRoot, 'Line').text = str(infoDict['lineno'])
    showTestFunctionSortLiterals = "0"
    if 'showTestFunctionSortingLiterals' in infoDict:
      shouldShowLiterals = mevis.MLAB.valueIsTrue(infoDict['showTestFunctionSortingLiterals'])
      showTestFunctionSortLiterals = "1" if shouldShowLiterals else "0"
    etree.SubElement(self.__xmlRoot, 'showTestFunctionSortingLiterals').text = showTestFunctionSortLiterals
    if 'preferredRenderer' in infoDict:
      etree.SubElement(self.__xmlRoot, 'preferredRenderer').text = infoDict['preferredRenderer']
 
    # Set the data directory. If not specified use default value.
    if u'dataDirectory' in infoDict:
      self._dataDirectory = infoDict['dataDirectory']
    else:
      self._dataDirectory = os.path.join(os.path.dirname(infoDict['file']), "Data")
 
    MLABpriv.clearLogState()
    MLABpriv.startLogCollect()
 
    if not (infoDict and "scriptFile" in infoDict):
      mevis.MLAB.logError(f"Missing scriptFile tag in TestCase {testCaseName}.")
    else:
      self._loadScriptFile(infoDict["scriptFile"])
      # we need to activate the TestCase, because building of the virtual functions will call into
      # TestHelper functions...
      self.activate()
 
      self._buildVirtualFunctions(isNewlyCreated)
 
      self.deactivate()
 
      etree.SubElement(self.__xmlRoot, 'Documentation').text = self._ctx.scriptVariable("__doc__")
 
    # create a virtual test function to show the setup errors
    if MLABpriv.hadLogWarnings() or MLABpriv.hadLogErrors():
      self.__createVirtualTestFunction("LoadTestCase", Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION, "Collects errors while the test case is loaded")
    else:
      self.__valid = True
 
    # create a virtual test function to show tear down errors
    self.__createVirtualTestFunction("UnloadTestCase", Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION, "Collects errors while the test case is unloaded")
 
    self.__stopCollectingAndStoreAsStartupLog()
  # --------------------------------------------------------------------------------------------}}}-
 
  def __createVirtualTestFunction(self, basename, name, documentation):
    functionsNode = self.__xmlRoot.find("TestFunctions")
    if functionsNode == None:
      functionsNode = etree.SubElement(self.__xmlRoot, "TestFunctions")
    functionXMLNode = etree.SubElement(functionsNode, "Function", basename=basename, name=name, is_disabled=str(False))
    etree.SubElement(functionXMLNode, "Documentation").text = documentation
 
  def _getTestCaseInfo(self, testCaseName):
    return mevis.MLABTestCaseDatabase.testCaseInfo(testCaseName)
 
  # -- def __del__ -----------------------------------------------------------------------------{{{-
  
  def __del__ (self):
    self.destroyContext()
  # --------------------------------------------------------------------------------------------}}}-
 
  def destroyContext(self):
    if self._ctx:
      try:
        self._ctx.remove()
      finally:
        self._ctx = None
 
  def activate(self):
    # Tell the TestHelper about us, but don't set a reference
    # to self to the TestHelper to avoid that it reference counts us:
    self._testHelper.setMacrosLogOnSuccess(True)
    self._testHelper.setTestCaseContext(self._ctx)
    self._testHelper.setTestCaseName(self._testCaseName)
    self._testHelper.setTestCaseDataDirectory(self._dataDirectory)
    self._testHelper.pushChangeSet()
 
  # --------------------------------------------------------------------------------------------}}}-
 
  def deactivate(self):
    # Reset the test helper. If activate()/deactivate() should need to be
    # called recursive in the the future, we would need to keep a stack here and in
    # activate().
    self._testHelper.setTestCaseContext(None)
    self._testHelper.setTestCaseName("")
    self._testHelper.setTestCaseDataDirectory("")
    self._testHelper.popChangeSet()
    self._testHelper.setMacrosLogOnSuccess(True)
 
  # --------------------------------------------------------------------------------------------}}}-
 
  def setTestCaseListener (self, testCaseListener):
    self.__testCaseListener = testCaseListener
 
  # -- def addExtraTestCaseResult ------------------------------------------------------------------{{{-
  def addExtraTestCaseResult(self, extraTestCaseResult):
    assert(self.__currentResultNode != None)
    assert(self.__currentResultNode != self.__xmlRoot)
    if self.__extraTestCasesResults == None:
      self.__extraTestCasesResults = etree.SubElement(self.__currentResultNode, "ExtraTestCasesResults")
    self.__extraTestCasesResults.append(extraTestCaseResult.toXML())
  # --------------------------------------------------------------------------------------------}}}-
 
  def _getTestFunction(self, funcName):
    return self._ctx.scriptVariable(funcName)
 
  # -- def __callTestFunction ------------------------------------------------------------------{{{-
  
  def __callTestFunction (self, funcName):
    if funcName in Utils.VIRTUAL_TEST_CASE_FUNCTIONS:
      return
 
    if not funcName in self.__functionDict:
      raise Exception("Unknown function!")
 
    changeSetLength = self._testHelper.pushChangeSet()
 
    callInfo = self.__functionDict[funcName]
 
    try:
      if callInfo:
        if type(callInfo) in (list, tuple):
          testFunction, testArguments = callInfo
          callWithKeywordArguments = False
          if len(testArguments) == 1 and isinstance(testArguments[0], dict):
            signature = inspect.signature(testFunction)
            try:
              # try to bind the test arguments as keyword arguments to the test function
              signature.bind(**testArguments[0])
              callWithKeywordArguments = True
            except TypeError:
              pass
          if callWithKeywordArguments:
            result = testFunction(**testArguments[0])
          else:
            result = testFunction(*testArguments)
        else:
          result = self._getTestFunction(callInfo)()
      else:
        result = self._getTestFunction(funcName)()
 
      # if the result is a coroutine, we schedule it and process events until it
      # has finished... This allows test functions to use the async keyword.
      if asyncio.iscoroutine(result):
        self.futureDone = False
        def futureDone(*args): self.futureDone = True
        future = asyncio.ensure_future(result)
        future.add_done_callback(futureDone)
        while not self.futureDone:
          QtCore.QCoreApplication.processEvents()
        future.result()
 
      if TestSupport.Logging.gStopped:
        raise CancelTestException
 
    except CancelTestException:
      self._testHelper.clearChangeSet()
      raise
    except TestSupport.Macros.AssertException:
      # an assertion has failed and we should stop execution,
      # but this is an expected exception, so we don't print it.
      pass
    except: # pylint: disable=W0702
      mevis.MLAB.logError(f"Exception occurred in {funcName}:")
      printTestFunctionException()
 
    # Clean up the environment.
    self._testHelper.popChangeSet()
    while self._testHelper.getChangeSetStackLength() > changeSetLength:
      mevis.MLAB.logError("You should pop your ChangeSets by yourself!")
      self._testHelper.popChangeSet()
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __stopCollectingAndStoreAsStartupLog ------------------------------------------------------------------------------{{{-
  
  def __stopCollectingAndStoreAsStartupLog (self):
    if MLABpriv.hadLogWarnings() or MLABpriv.hadLogErrors():
      self.__collectedStartupLog = MLABpriv.collectedLog()
      self.__collectedStartupErrorsState = MLABpriv.hadLogErrors()
      self.__collectedStartupWarningsState = MLABpriv.hadLogWarnings()
    else:
      self.__collectedStartupLog = None
      self.__collectedStartupErrorsState = False
      self.__collectedStartupWarningsState = False
    MLABpriv.stopLogCollect()
 
  # -- def _loadScriptFile --------------------------------------------------------------------{{{-
  
  def _loadScriptFile (self, filename):
    self._ctx = self._testHelper.getGlobalContext().addModule(filename)
    if self._ctx:
      # set dynamic property so that MeVisLab and MATE know that this is a TestCase
      self._ctx.setProperty("IsTestCase", True)
      self._ctx.setProperty("TestCaseName", self._testCaseName)
 
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def callFieldValueTestCase --------------------------------------------------------------{{{-
  
  def callFieldValueTestCase (self, filename, testcase):
    fieldValueTestCaseSet = FieldValueTestCaseSet(self._ctx)
    if not fieldValueTestCaseSet.load(filename):
      mevis.MLAB.logError(f"Failed ot load field-value test ({filename}).")
    else:
      testCase = fieldValueTestCaseSet.get(testcase)
      testCase.applyParameterization(TestHelper.getInstance().getChangeSet(), verbose=True)
      if not testCase.verifyExpectedResults(verbose=True):
        mevis.MLAB.logError("Failed to verify expected results.")
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __createNormalTestNode ----------------------------------------------------------------------{{{-
  
  def __createNormalTestNode (self, virtualFunctionName, functionName, functionBaseName, functionDict, isDisabled):
    retVal = True
    if not functionBaseName in functionDict["names"]:
      xmlNode = self.__getFunctionInfo(functionBaseName, functionName, isDisabled=isDisabled, sourceName=functionName)
      functionDict["names"].append(functionBaseName)
      functionDict["nodes"][virtualFunctionName] = xmlNode
    else:
      mevis.MLAB.logError(f"Function with name {functionBaseName} already defined!")
      retVal = False
    return retVal
  # --------------------------------------------------------------------------------------------}}}-
 
  def _callGroupFunction(self, groupFunctionName):
    return self._ctx.call(groupFunctionName, [])
 
  # -- def __createTestGroupNode ---------------------------------------------------------------{{{-
  
  def __createTestGroupNode (self, groupFunctionName, orderString, groupName, functionDict, isDisabled):
    if groupName in functionDict["names"]:
      mevis.MLAB.logError(f"A test group with name {groupName} already exists. No duplicate names allowed.")
      return
 
    # Get the field-value test case set file.
    functionList = self._callGroupFunction(groupFunctionName)
    if not functionList:
      mevis.MLAB.logError(f"Failed to build virtual functions for the {groupFunctionName} test.")
      return
 
    if type(functionList) not in (list, tuple):
      mevis.MLAB.logError(f"The test group function ({groupFunctionName}) must return a list or tuple with function objects!")
      return
 
    virtualGroupName = f"TEST{orderString}_{groupName}"
    groupXMLNode = etree.Element("GroupNode", name=virtualGroupName, basename=groupName, type="TestGroup", sourceName=groupFunctionName)
    etree.SubElement(groupXMLNode, "Documentation").text = self._ctx.scriptVariable(f"{groupFunctionName}.__doc__")
 
    nameToFunctionMap = {}
    for function in functionList:
      # allow function names as string
      if isinstance(function, str):
        function = self._ctx.scriptVariable(function)
      nameToFunctionMap[function.__name__] = function
 
    for functionName in sorted(nameToFunctionMap.keys()):
      function = nameToFunctionMap[functionName]
 
      functionNodeName = functionName
      if functionNodeName.startswith('DISABLED_'):
        functionNodeName = functionNodeName[len('DISABLED_'):]
 
      if functionNodeName not in functionDict["nodes"] or functionDict["nodes"][functionNodeName].tag != "Function":
        mevis.MLAB.logError(f"Failed to compile function list. Given function {functionNodeName} not a test function "
                            f"or function already part of a test function group.")
        return False
 
      # Update function info, i.e. move the function node to the group node. The
      # dictionary with the virtual function names must be updated too.
      virtualFunctionName = (f"{virtualGroupName}_"
                             f"{functionNodeName[4:] if not functionNodeName.startswith('DISABLED_') else functionNodeName[13:]}"
                             )
 
      functionNode = functionDict["nodes"][functionNodeName]
      functionNode.set("name", virtualFunctionName)
      # mark function as disabled if the whole group is disabled; otherwise, keep the current state of the function
      # this way explicitly disabled functions stay disabled
      if isDisabled:
        functionNode.set("is_disabled", str(isDisabled))
 
      # Remove function from dictionary of nodes. It will be part of the group!
      del functionDict["nodes"][functionNodeName]
 
      # Remove old virtual function
      self.__functionDict.pop(functionName)
      self.__functionDict[virtualFunctionName] = functionName
 
      groupXMLNode.append(functionNode)
 
    functionDict["names"].append(groupName)
    functionDict["nodes"][virtualGroupName] = groupXMLNode
    return True
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __createFieldValueTestNode ----------------------------------------------------------{{{-
  
  def __createFieldValueTestNode (self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
    # Find all the field-value test case functions, expand them and add the
    # single items to the virtual function list.
    if groupName in functionDict["names"]:
      mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
      return
 
    # Get the field-value test case set file.
    self._testHelper.pushChangeSet()
    retVal = self._ctx.call(functionName, [])
    self._testHelper.popChangeSet()
 
    filename, testCaseList = retVal if type(retVal) in (list, tuple) else (retVal, None)
 
    groupXMLNode = etree.Element("GroupNode", name=virtualFunctionBaseName, basename=groupName, type="FieldValueTest", sourceName=functionName)
    etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
    # Keep in mind that duplicate names aren't allowed in field-value test case
    # sets either.
    functionDict["names"].append(groupName)
    functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
 
    fvtcs = FieldValueTestCaseSet(self._ctx)
    if not fvtcs.load(filename):
      mevis.MLAB.logError(f"Failed to load the field-value test case set ({filename})!")
      return False
 
    if not testCaseList:
      for testcase in sorted(fvtcs.getList()):
        virtualFunctionName = f"{virtualFunctionBaseName}_{testcase}"
        groupXMLNode.append(self.__getFunctionInfo(testcase, virtualFunctionName, (
          self.callFieldValueTestCasecallFieldValueTestCase, (filename, testcase)), isDisabled=isDisabled, sourceName=functionName))
    else:
      availableTestCaseList = fvtcs.getList()
      for testcase in sorted(testCaseList):
        if testcase in availableTestCaseList:
          virtualFunctionName = f"{virtualFunctionBaseName}_{testcase}"
          groupXMLNode.append(self.__getFunctionInfo(testcase, virtualFunctionName, (
            self.callFieldValueTestCasecallFieldValueTestCase, (filename, testcase)), isDisabled=isDisabled, sourceName=functionName))
        else:
          mevis.MLAB.logError(f"The field-value test case {testcase} is unknown!")
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __createIterativeTestNode -----------------------------------------------------------{{{-
  
  def __createIterativeTestNode (self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
    # Find all iterative test functions, expand them and add the single items
    # to the virtual function list.
    if groupName in functionDict["names"]:
      mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
      return
 
    # Get the iterations and the actual function to call.
    self._testHelper.pushChangeSet()
    retVal = self._ctx.call(functionName, [])
    self._testHelper.popChangeSet()
 
    try:
      iterator, function = retVal
    except: # pylint: disable=W0702
      mevis.MLAB.logError(f"The iterative test function ({functionName}) must return a list or dictionary"
                          f" with parameters and a function to call!")
      return
 
    groupXMLNode = etree.Element("GroupNode", name=virtualFunctionBaseName, basename=groupName, type="IterativeTest", sourceName=functionName)
    etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
    # Duplicate item nodes are not possible by design (either an index is used
    # or a dictionary).
    functionDict["names"].append(groupName)
    functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
    regExpFunctionNames = re.compile(r"^[.\w]+$")
 
    # Create virtual functions for given iterative test.
    if isinstance( iterator, (list, tuple) ):
      ctr = 0
      # Let the index used to identify the function be zero padded.
      justificationWidth = len(unicode(len(iterator)-1))
      for item in iterator:
        # The zero padded identifier.
        virtualFunctionId = unicode(ctr).rjust(justificationWidth, "0")
        args = [item] if (type(item) not in (list, tuple)) else item
        virtualFunctionName = f"{virtualFunctionBaseName}_{virtualFunctionId}"
        if regExpFunctionNames.match(virtualFunctionName):
          groupXMLNode.append(self.__getFunctionInfo(virtualFunctionId, virtualFunctionName, (function, args), isDisabled=isDisabled, sourceName=functionName))
        else:
          mevis.MLAB.logError(f"The iterative test function ({functionName}) defines non-alphanumeric virtual "
                              f"function name {virtualFunctionName}!")
        ctr += 1
    elif isinstance( iterator, dict ):
      for item in sorted(iterator.keys()):
        args = [iterator[item]] if (type(iterator[item]) not in (list, tuple)) else iterator[item]
        virtualFunctionName = f"{virtualFunctionBaseName}_{item}"
        if regExpFunctionNames.match(virtualFunctionName):
          groupXMLNode.append(self.__getFunctionInfo(unicode(item), virtualFunctionName, (function, args), isDisabled=isDisabled, sourceName=functionName))
        else:
          mevis.MLAB.logError(f"The iterative test function ({functionName}) defines non-alphanumeric virtual "
                              f"function name {virtualFunctionName}!")
    else:
      mevis.MLAB.logError(f"The iterative test function ({functionName}) must return a list or dictionary with parameters!")
 
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __createUnitTestWrapperNode -----------------------------------------------------------{{{-
  
  def __createUnitTestWrapperNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
    # Find all unit test wrapper functions, expand them and add the single items
    # to the virtual function list.
 
    if groupName in functionDict["names"]:
      mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
      return
 
    # Get the unit test and the actual function to call.
    self._testHelper.pushChangeSet()
    suite = self._ctx.call(functionName, [])
    self._testHelper.popChangeSet()
 
    if not isinstance( suite, unittest.TestSuite ):
      mevis.MLAB.logError(f"The unit test wrapper function ({functionName}) must return a unittest.TestSuite!")
      return
 
    groupXMLNode = etree.Element("GroupNode", name=virtualFunctionBaseName, basename=groupName, type="UnitTestWrapper", sourceName=functionName)
    etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
    # Duplicate item nodes are not possible by design (either an index is used
    # or a dictionary).
    functionDict["names"].append(groupName)
    functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
 
    def _isSkipped(test):
      isClassSkipped = getattr(test.__class__, "__unittest_skip__", False)
      testMethod = getattr(test, test._testMethodName) # pylint: disable=W0212
      isMethodSkipped = getattr(testMethod, "__unittest_skip__", False)
      return isClassSkipped or isMethodSkipped
 
    # Create virtual functions for given unit test wrapper
    def _unpackSuite(suite):
      allTests = []
      for test in suite._tests: # pylint: disable=W0212
        if isinstance( test, unittest.TestSuite ):
          allTests.extend( _unpackSuite( test ) )
        else:
          if not _isSkipped(test):
            allTests.append( test )
      return allTests
    allTests = _unpackSuite( suite )
 
    def _makeName(test):
      testCaseName = test.__class__.__name__
      testFunctionName = test._testMethodName # pylint: disable=W0212
      return '{0}_{1}'.format( testCaseName, testFunctionName )
    allTests.sort(key=_makeName)
    testNames = list(map( _makeName, allTests ))
 
    # It could be smarter to integrate this more deeply into the callTestFunction part.
    def _runUnitTestFunction(testFunction):
      testResult = UnitTestResult()
      testResult._previousTestClass = self.__previousTestClass
      if self.__dummyTestSuite is None:
        self.__dummyTestSuite = unittest.TestSuite()
      # handle setup of class and module fixtures (copied from unittest.TestSuite):
      self.__dummyTestSuite._tearDownPreviousClass(testFunction, testResult)
      self.__dummyTestSuite._handleModuleFixture(testFunction, testResult)
      self.__dummyTestSuite._handleClassSetUp(testFunction, testResult)
      self.__previousTestClass = testFunction.__class__
      # run test:
      testFunction.run( testResult )
      testResult.logResult( 'Test was successful' )
 
    for testName, testFunction in zip( testNames, allTests ):
      testMethod = getattr(testFunction, testFunction._testMethodName)
      funcXMLNode = self.__getFunctionInfo(testName, '{}_{}'.format(virtualFunctionBaseName, testName),
                                           (_runUnitTestFunction, [testFunction]), isDisabled=isDisabled)
      if hasattr(testMethod, "__func__"):  # testMethod is really a method
        funcXMLNode.attrib["sourceFilename"] = testMethod.__func__.__code__.co_filename
        funcXMLNode.attrib["sourceLine"] = str(testMethod.__func__.__code__.co_firstlineno)
      elif hasattr(testMethod, "__code__"):  # testMethod is actually a function
        funcXMLNode.attrib["sourceFilename"] = testMethod.__code__.co_filename
        funcXMLNode.attrib["sourceLine"] = str(testMethod.__code__.co_firstlineno)
      else:  # None or unsupported type - should we raise here?
        funcXMLNode.attrib["sourceFilename"] = ""
        funcXMLNode.attrib["sourceLine"] = "0"
      groupXMLNode.append(funcXMLNode)
 
  def _callableFunctions(self):
    return self._ctx.callableFunctions()
 
  # -- def __buildVirtualFunctions ---------------------------------------------------------------------{{{-
  
  def _buildVirtualFunctions (self, isNewlyCreated):
    # A list containing the found function names (to prevent duplicates).
    functionInformationDict = { "names":[], "nodes":{} }
 
    regExpDict = { "TestFunction":    re.compile(r"^(DISABLED_)?TEST(\d*)_(\w+)$"),
                   "TestGroup":       re.compile(r"^(DISABLED_)?GROUP(\d*)_(\w+)$"),
                   "FieldValueTest":  re.compile(r"^(DISABLED_)?FIELDVALUETEST(\d*)_(\w+)$"),
                   "IterativeTest":   re.compile(r"^(DISABLED_)?ITERATIVETEST(\d*)_(\w+)$"),
                   "UnitTestWrapper": re.compile(r"^(DISABLED_)?UNITTEST(\d*)_(\w+)$") }
 
    # The result node.
    functionsNode = etree.SubElement(self.__xmlRoot, "TestFunctions")
 
    # Check the optional list of disabled test functions (it is defined by the disabledTestFunction() decorator)
    disabledFunctions = self._ctx.scriptVariable('MLABTC_DISABLED_TEST_FUNCTIONS') or []
 
    # The dictionary of all test functions sorted by type.
    functionDict = { "TestFunction":[], "TestGroup":[], "FieldValueTest":[], "IterativeTest":[], "UnitTestWrapper":[] }
    for functionName in self._callableFunctions():
        for prefix, functionList in (("TEST",           functionDict["TestFunction"]),
                                     ("GROUP",          functionDict["TestGroup"]),
                                     ("FIELDVALUETEST", functionDict["FieldValueTest"]),
                                     ("ITERATIVETEST",  functionDict["IterativeTest"]),
                                     ("UNITTEST",       functionDict["UnitTestWrapper"]) ):
            if functionName.startswith(prefix) or functionName.startswith("DISABLED_" + prefix):
                functionList.append(functionName)
 
    for functionName in sorted(functionDict["TestFunction"]):
      mObj = regExpDict["TestFunction"].match(functionName)
      if not mObj:
        mevis.MLAB.logError(f"Name of test function invalid ({functionName})")
      else:
        disabledString, orderString, functionBaseName = mObj.groups()
        isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
        fname = f"TEST{orderString}_{functionBaseName}"
        self.__createNormalTestNode(fname, functionName, functionBaseName, functionInformationDict, isDisabled=isDisabled)
 
    for functionName in sorted(functionDict["TestGroup"]):
      mObj = regExpDict["TestGroup"].match(functionName)
      if not mObj:
        mevis.MLAB.logError(f"Name of test group invalid ({functionName})")
      else:
        disabledString, orderString, groupName = mObj.groups()
        isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
        self.__createTestGroupNode(functionName, orderString, groupName, functionInformationDict, isDisabled=isDisabled)
 
    for functionName in sorted(functionDict["FieldValueTest"]):
      mObj = regExpDict["FieldValueTest"].match(functionName)
      if not mObj:
        mevis.MLAB.logError(f"Name of field value test invalid ({functionName})")
      else:
        disabledString, orderString, groupName = mObj.groups()
        isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
        fname = f"TEST{orderString}_{groupName}"
        self.__createFieldValueTestNode(fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled)
 
    for functionName in sorted(functionDict["IterativeTest"]):
      mObj = regExpDict["IterativeTest"].match(functionName)
      if not mObj:
        mevis.MLAB.logError(f"Name of iterative test invalid ({functionName})")
      else:
        disabledString, orderString, groupName = mObj.groups()
        isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
        fname = f"TEST{orderString}_{groupName}"
        self.__createIterativeTestNode(fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled)
 
    for functionName in sorted(functionDict["UnitTestWrapper"]):
      mObj = regExpDict["UnitTestWrapper"].match(functionName)
      if not mObj:
        mevis.MLAB.logError(f"Name of unit test wrapper invalid ({functionName})")
      else:
        disabledString, orderString, groupName = mObj.groups()
        isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
        fname = f"TEST{orderString}_{groupName}"
        self.__createUnitTestWrapperNode(fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled)
 
    # Append nodes in correct order. Essential for other methods to work
    # properly.
    keys = sorted(functionInformationDict["nodes"].keys())
    if len(keys)>0:
      for functionName in keys:
        functionsNode.append(functionInformationDict["nodes"][functionName])
    elif not isNewlyCreated:
      mevis.MLAB.logError(f"No test functions found in TestCase {self._testCaseName}!")
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __getFunctionInfo -------------------------------------------------------------------{{{-
  
  def __getFunctionInfo (self, functionBaseName, functionName, calledFunction=None, isDisabled=False, sourceName=''):
    self.__functionDict.setdefault(functionName, calledFunction)
    functionXMLNode = etree.Element("Function", basename=functionBaseName, name=functionName, sourceName=sourceName, is_disabled=str(isDisabled))
    documentation = self.getDocumentation(functionName)
    etree.SubElement(functionXMLNode, "Documentation").text = documentation if documentation else ""
    return functionXMLNode
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def isValid -----------------------------------------------------------------------------{{{-
  
  def isValid (self):
    return self.__valid
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def hasNetwork --------------------------------------------------------------------------{{{-
  
  def hasNetwork ( self ):
    return ( os.path.exists( self.networkFileName() ) )
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def networkFileName --------------------------------------------------------------------------{{{-
  
  def networkFileName ( self ):
    return ( self._ctx.network().filename() )
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def openFiles ---------------------------------------------------------------------------{{{-
  
  def openFiles (self):
    typeName = self.getName()
    for fileName in self._ctx.getScriptSourceFiles():
      mevis.MLAB.priv().openRelatedFile(typeName, fileName)
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def openNetwork -------------------------------------------------------------------------{{{-
  
  def openNetwork (self):
    if self._ctx:
      mevis.MLAB.priv().showIDE()
      self._ctx.showInternalNetwork()
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getType -----------------------------------------------------------------------------{{{-
  
  def getType (self):
    return self._testCaseType
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getModule -----------------------------------------------------------------------------{{{-
  
  def getModule (self):
    return self._ctx
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getName -----------------------------------------------------------------------------{{{-
  
  def getName (self):
    return self._testCaseName
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getListOfTestFuncs ------------------------------------------------------------------{{{-
  
  def getListOfTestFuncs (self):
    return sorted(self.__functionDict.keys())
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getPackage --------------------------------------------------------------------------{{{-
  
  def getPackage (self):
    return self.__xmlRoot.find("Package").text
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getTestCaseNode ---------------------------------------------------------------------{{{-
  
  def getTestCaseNode (self):
    return self.__xmlRoot
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __callSetUpTestCase -----------------------------------------------------------------{{{-
  
  def __callSetUpTestCase (self):
    if u"setUpTestCase" in self._ctx.callableFunctions():
      self._ctx.call(u"setUpTestCase", [])
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def __callTearDownTestCase --------------------------------------------------------------{{{-
  
  def __callTearDownTestCase (self):
    if u"tearDownTestCase" in self._ctx.callableFunctions():
      self._ctx.call(u"tearDownTestCase", [])
  # --------------------------------------------------------------------------------------------}}}-
 
  def __tearDown(self):
    if self.__setupTestCaseCalled:
      mevis.MLAB.log("Calling tearDown() function...")
      self.__callTearDownTestCase()
    if self.__dummyTestSuite:
      # unittest support: clean up after running unittest.TestCases
      testResult = UnitTestResult()
      testResult._previousTestClass = self.__previousTestClass
      self.__dummyTestSuite._tearDownPreviousClass(None, testResult)
      self.__dummyTestSuite._handleModuleTearDown(testResult)
      testResult.logResult('')
      self.__previousTestClass = None
      self.__dummyTestSuite = None
    if TestSupport.Logging.gStopped:
      raise CancelTestException
    else:
      mevis.MLAB.log("Destroying test case context...")
      self.destroyContext()
 
  # -- def run ---------------------------------------------------------------------------------{{{-
  
  def run (self, resultNode):
 
    self.activate()
 
    TestSupport.Logging.gStopped = False
 
    logHandler = self._testHelper.getLogHandler()
 
    availTestFunctions = self.getListOfTestFuncs()
 
    # Build dict mapping test function names to xml nodes.
    funcList = resultNode.findall('.//Function')
    length = len(funcList)
 
    self.__currentResultNode = resultNode
 
    # Make sure the virtual test function for loading the test case is the first, if it exists.
    # This ensures that initial errors/warnings are appended to this function. Accordingly
    # the unload test case function must be last.
    loadTestCaseFunctionHandled = False
    unloadTestCaseFunctionHandled = False
    for i in range(0, length):
      funcNode = funcList[i]
      funcName = funcNode.get("name")
      if not loadTestCaseFunctionHandled and funcName == Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION:
        del funcList[i]
        funcList.insert(0, funcNode)
        loadTestCaseFunctionHandled = True
      elif not unloadTestCaseFunctionHandled and funcName == Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION:
        del funcList[i]
        funcList.append(funcNode)
        unloadTestCaseFunctionHandled = True
      if loadTestCaseFunctionHandled and unloadTestCaseFunctionHandled:
        break
 
    self.__setupTestCaseCalled = False
 
    # Iterate over all test functions (sorted by name) that should be tested.
    # The iteration with an index is required to call setUpTestCase and
    # tearDownTestCase properly.
    for i in range(0, length):
      funcNode = funcList[i]
      funcName = funcNode.get("name")
 
      isLastTestFunction = i == length-1
      data = RunTestFunctionData(funcNode, funcName, logHandler, isLastTestFunction)
 
      if not self.__prepareRunTestFunction(data, availTestFunctions):
        continue
 
      self.__runTestFunction(data)
      self.__finishRunTestFunction(data)
      if data.cancelled:
        # Do NOT call __tearDown() when 'Stop on error' flag is set and an error occured.
        if not (TestSupport.Logging.gStopOnFirstError and logHandler.numErrors() > 0):
          self.__tearDown()
        break
 
    self.deactivate()
 
  def __prepareRunTestFunction(self, data, availTestFunctions):
    if self.__perFunctionResultDirectory:
      funcNameForResultDir = data.funcName
    else:
      funcNameForResultDir = None
    self._testHelper.setTestCaseResultDirectory(self.getName().replace("::", os.sep), funcNameForResultDir)
 
    shouldRunTestFunction = True
 
    # Verify that a function with the given name exists.
    if data.funcName not in availTestFunctions and data.funcName not in Utils.VIRTUAL_TEST_CASE_FUNCTIONS:
      mevis.MLAB.logError(f"Function {data.funcName} does not exist in test case {self.getName()}!")
      status = Utils.TEST_STATUS_FUNCTION_NOT_FOUND
      data.funcNode.set('status', str(status))
      shouldRunTestFunction = False
    return shouldRunTestFunction
 
  def __runTestFunction(self, data):
    data.started = time.time()
 
    try:
      data.logHandler.startTestFunction(self.getType(), self.getName(), data.funcName)
      if self.__testCaseListener:
        self.__testCaseListener.startTestCaseFunction(self.getType(), self.getName(), data.funcName)
 
      MLABpriv.clearLogState()
 
      # append earlier testcase error/warnings to the first test function that is executed
      self.__appendCollectedStartupLog(data.logHandler)
 
      self.__setupTestCaseAndRunTestFunction(data)
 
      if data.funcName == Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION:
        self.__tearDown()
      if TestSupport.Logging.gStopped:
        raise CancelTestException
    except CancelTestException:
      data.cancelled = True
 
  def __finishRunTestFunction(self, data):
    for extraTestCaseResult in self._testHelper.takeExtraTestCaseResults():
      self.addExtraTestCaseResult(extraTestCaseResult)
 
    # process events so that any delayed messages are also part of the test function
    mevis.MLAB.processEvents()
    # force messages from other threads to be printed inside of this test function execution
    MLABpriv.flushPendingMessages()
 
    duration = time.time()-data.started
 
    # Gather status and additional information.
    status = Utils.TEST_STATUS_OK
    if MLABpriv.hadLogWarnings() or self.__collectedStartupWarningsState: status = Utils.TEST_STATUS_WARNING
    if MLABpriv.hadLogErrors() or self.__collectedStartupErrorsState:   status = Utils.TEST_STATUS_ERROR
    self.__collectedStartupErrorsState = False
    self.__collectedStartupWarningsState = False
    data.logHandler.stopTestFunction(status, duration)
    if self.__testCaseListener:
      self.__testCaseListener.endTestCaseFunction(status, duration)
 
    if data.cancelled:
      status = Utils.TEST_STATUS_CANCELLED
    data.funcNode.set('status', str(status))
    data.funcNode.set('duration', str(duration))
    data.funcNode.append(data.logHandler.getMessages())
 
  def __appendCollectedStartupLog(self, logHandler):
    if self.__collectedStartupLog:
      log = self.__collectedStartupLog
      msgs = log.split('<br>')
      if len(msgs):
        mevis.MLAB.log("TestCase setup errors/warnings:")
        for msg in msgs:
          if msg!="":
            logHandler.appendMessage(msg)
        mevis.MLAB.log("TestCase execution:")
      self.__collectedStartupLog = None
 
  def __setupTestCaseAndRunTestFunction(self, data):
    if data.funcNode.get('is_disabled', str(False)) == str(False):
      if not self.__setupTestCaseCalled:
        self.__setupTestCaseCalled = True
        self.__callSetUpTestCase()
        if TestSupport.Logging.gStopped:
          raise CancelTestException
      mevis.MLAB.log(f"Calling test function {data.funcName}()")
      self.__callTestFunction(data.funcName)
    else:
      mevis.MLAB.log(f"Skipping disabled test function {data.funcName}()")
 
  # --------------------------------------------------------------------------------------------}}}-
 
  # -- def getDocumentation --------------------------------------------------------------------{{{-
  
  def getDocumentation (self, functionName):
    docu = self._ctx.scriptVariable(f"{functionName}.__doc__")
    testDocu = docu if docu else ""
    return re.sub(r"(?m)^\s+", "", testDocu)
  # --------------------------------------------------------------------------------------------}}}-
# ----------------------------------------------------------------------------------------------}}}-