TestCase.py

Go to the documentation of this file.

#
# Copyright 2009, MeVis Medical Solutions AG
#
# The user may use this file in accordance with the license agreement provided with
# the Software or, alternatively, in accordance with the terms contained in a
# written agreement between the user and MeVis Medical Solutions AG.
#
# For further information use the contact form at https://www.mevislab.de/contact
#
 
 
 
# -- system imports ----------------------------------------------------------------------------{{{-
import inspect
import os
import re
import sys
import time
import traceback
import unittest
import asyncio
from PythonQt import QtCore
 
if sys.version_info.major >= 3:
    unicode = str
 
import xml.etree.cElementTree as etree
 
# ----------------------------------------------------------------------------------------------}}}-
 
# -- local imports -----------------------------------------------------------------------------{{{-
import mevis
from TestSupport.FieldValueTests import FieldValueTestCaseSet
 
from TestSupport.TestHelper import TestHelper, CancelTestException
from . import Utils
import TestSupport.Logging
import TestSupport.Macros
 
# ----------------------------------------------------------------------------------------------}}}-
 
# -- local imports -----------------------------------------------------------------------------{{{-
MLABpriv = mevis.MLAB.priv()
# ----------------------------------------------------------------------------------------------}}}-
 
 
# Prints the exception that occurred during a test function.
def printTestFunctionException():
    exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
    tb = exceptionTraceback.tb_next
    traceback.print_tb(tb, limit=None, file=sys.stderr)
    lines = traceback.format_exception_only(exceptionType, exceptionValue)
    for line in lines:
        sys.stderr.write(line)
 
 
class RunTestFunctionData:
    def __init__(self, funcNode, funcName, logHandler, isLastTestFunction):
        self.funcNode = funcNode
        self.funcName = funcName
        self.logHandler = logHandler
        self.isLastTestFunction = isLastTestFunction
        self.started = None
        self.cancelled = False
 
 
class UnitTestResult(unittest.TestResult):
 
    # deactivate stdout/stderr redirection, the output is captured by the TestCenter anyway
    def _setupStdout(self):
        pass
 
    def _restoreStdout(self):
        pass
 
    def logResult(self, successMsg):
        if self.wasSuccessful() and successMsg:
            mevis.MLAB.log(successMsg)
        if self.errors:
            for error in self.errors:
                mevis.MLAB.logError(error[1])
        if self.failures:
            for failure in self.failures:
                mevis.MLAB.logError(failure[1])
 
 
# -- class TestCase ----------------------------------------------------------------------------{{{-
 
class TestCase:
    # -- member variables ------------------------------------------------------------------------{{{-
    
    _ctx = None
    
    _testHelper = None
 
    
    _testCaseName = None
    
    _testCaseType = None
 
    
    __xmlRoot = None
    
    __functionDict = None
 
    
    __valid = None
 
    # XML node for extra test case results
    __extraTestCasesResults = None
    
    __currentResultNode = None
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __init__ ----------------------------------------------------------------------------{{{-
    
    def __init__(self, testCaseName, isNewlyCreated=False):
        self._testCaseName = testCaseName
        self._testHelper = TestHelper.getInstance()
        self.__testCaseListener = None
 
        self.__collectedStartupLog = None
        self.__collectedStartupErrorsState = False
        self.__collectedStartupWarningsState = False
        self.__setupTestCaseCalled = False
 
        self.__valid = False
        self.__functionDict = {}
 
        
        self.__dummyTestSuite = None
 
        
        self.__previousTestClass = None
 
        infoDict = self._getTestCaseInfo(testCaseName)
 
        # Get the basic test case information and store them in the XML node.
        self.__xmlRoot = etree.Element("TestCase", name=self._testCaseName, type=self._testCaseType)
        self.__xmlRoot.set("timeout", infoDict.get("timeout", "0"))
        self.__perFunctionResultDirectory = mevis.MLAB.valueIsTrue(infoDict.get("perFunctionResultDirectory", "1"))
 
        etree.SubElement(self.__xmlRoot, "Package").text = infoDict["package"]
        etree.SubElement(self.__xmlRoot, "Author").text = infoDict["author"]
        etree.SubElement(self.__xmlRoot, "Maintainer").text = infoDict.get("maintainer", "")
        etree.SubElement(self.__xmlRoot, "Comment").text = infoDict.get("comment", "")
        etree.SubElement(self.__xmlRoot, "File").text = infoDict["file"]
        etree.SubElement(self.__xmlRoot, "Line").text = str(infoDict["lineno"])
        showTestFunctionSortLiterals = "0"
        if "showTestFunctionSortingLiterals" in infoDict:
            shouldShowLiterals = mevis.MLAB.valueIsTrue(infoDict["showTestFunctionSortingLiterals"])
            showTestFunctionSortLiterals = "1" if shouldShowLiterals else "0"
        etree.SubElement(self.__xmlRoot, "showTestFunctionSortingLiterals").text = showTestFunctionSortLiterals
        if "preferredRenderer" in infoDict:
            etree.SubElement(self.__xmlRoot, "preferredRenderer").text = infoDict["preferredRenderer"]
 
        # Set the data directory. If not specified use default value.
        if "dataDirectory" in infoDict:
            self._dataDirectory = infoDict["dataDirectory"]
        else:
            self._dataDirectory = os.path.join(os.path.dirname(infoDict["file"]), "Data")
 
        MLABpriv.clearLogState()
        MLABpriv.startLogCollect()
 
        if not (infoDict and "scriptFile" in infoDict):
            mevis.MLAB.logError(f"Missing scriptFile tag in TestCase {testCaseName}.")
        else:
            self._loadScriptFile(infoDict["scriptFile"])
            # we need to activate the TestCase, because building of the virtual functions will call into
            # TestHelper functions...
            self.activate()
 
            self._buildVirtualFunctions(isNewlyCreated)
 
            self.deactivate()
 
            etree.SubElement(self.__xmlRoot, "Documentation").text = self._ctx.scriptVariable("__doc__")
 
        # create a virtual test function to show the setup errors
        if MLABpriv.hadLogWarnings() or MLABpriv.hadLogErrors():
            self.__createVirtualTestFunction(
                "LoadTestCase",
                Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION,
                "Collects errors while the test case is loaded",
            )
        else:
            self.__valid = True
 
        # create a virtual test function to show tear down errors
        self.__createVirtualTestFunction(
            "UnloadTestCase",
            Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION,
            "Collects errors while the test case is unloaded",
        )
 
        self.__stopCollectingAndStoreAsStartupLog()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def __createVirtualTestFunction(self, basename, name, documentation):
        functionsNode = self.__xmlRoot.find("TestFunctions")
        if functionsNode == None:
            functionsNode = etree.SubElement(self.__xmlRoot, "TestFunctions")
        functionXMLNode = etree.SubElement(
            functionsNode, "Function", basename=basename, name=name, is_disabled=str(False)
        )
        etree.SubElement(functionXMLNode, "Documentation").text = documentation
 
    def _getTestCaseInfo(self, testCaseName):
        return mevis.MLABTestCaseDatabase.testCaseInfo(testCaseName)
 
    # -- def __del__ -----------------------------------------------------------------------------{{{-
    
    def __del__(self):
        self.destroyContext()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def destroyContext(self):
        if self._ctx:
            try:
                self._ctx.remove()
            finally:
                self._ctx = None
 
    def activate(self):
        # Tell the TestHelper about us, but don't set a reference
        # to self to the TestHelper to avoid that it reference counts us:
        self._testHelper.setMacrosLogOnSuccess(True)
        self._testHelper.setTestCaseContext(self._ctx)
        self._testHelper.setTestCaseName(self._testCaseName)
        self._testHelper.setTestCaseDataDirectory(self._dataDirectory)
        self._testHelper.pushChangeSet()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def deactivate(self):
        # Reset the test helper. If activate()/deactivate() should need to be
        # called recursive in the the future, we would need to keep a stack here and in
        # activate().
        self._testHelper.setTestCaseContext(None)
        self._testHelper.setTestCaseName("")
        self._testHelper.setTestCaseDataDirectory("")
        self._testHelper.popChangeSet()
        self._testHelper.setMacrosLogOnSuccess(True)
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def setTestCaseListener(self, testCaseListener):
        self.__testCaseListener = testCaseListener
 
    # -- def addExtraTestCaseResult ------------------------------------------------------------------{{{-
    def addExtraTestCaseResult(self, extraTestCaseResult):
        assert self.__currentResultNode != None
        assert self.__currentResultNode != self.__xmlRoot
        if self.__extraTestCasesResults == None:
            self.__extraTestCasesResults = etree.SubElement(self.__currentResultNode, "ExtraTestCasesResults")
        self.__extraTestCasesResults.append(extraTestCaseResult.toXML())
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def _getTestFunction(self, funcName):
        return self._ctx.scriptVariable(funcName)
 
    # -- def __callTestFunction ------------------------------------------------------------------{{{-
    
    def __callTestFunction(self, funcName):
        if funcName in Utils.VIRTUAL_TEST_CASE_FUNCTIONS:
            return
 
        if not funcName in self.__functionDict:
            raise Exception("Unknown function!")
 
        changeSetLength = self._testHelper.pushChangeSet()
 
        callInfo = self.__functionDict[funcName]
 
        try:
            if callInfo:
                if type(callInfo) in (list, tuple):
                    testFunction, testArguments = callInfo
                    callWithKeywordArguments = False
                    if len(testArguments) == 1 and isinstance(testArguments[0], dict):
                        signature = inspect.signature(testFunction)
                        try:
                            # try to bind the test arguments as keyword arguments to the test function
                            signature.bind(**testArguments[0])
                            callWithKeywordArguments = True
                        except TypeError:
                            pass
                    if callWithKeywordArguments:
                        result = testFunction(**testArguments[0])
                    else:
                        result = testFunction(*testArguments)
                else:
                    result = self._getTestFunction(callInfo)()
            else:
                result = self._getTestFunction(funcName)()
 
            # if the result is a coroutine, we schedule it and process events until it
            # has finished... This allows test functions to use the async keyword.
            if asyncio.iscoroutine(result):
                self.futureDone = False
 
                def futureDone(*args):
                    self.futureDone = True
 
                future = asyncio.ensure_future(result)
                future.add_done_callback(futureDone)
                while not self.futureDone:
                    QtCore.QCoreApplication.processEvents()
                future.result()
 
            if TestSupport.Logging.gStopped:
                raise CancelTestException
 
        except CancelTestException:
            self._testHelper.clearChangeSet()
            raise
        except TestSupport.Macros.AssertException:
            # an assertion has failed and we should stop execution,
            # but this is an expected exception, so we don't print it.
            pass
        except:  # pylint: disable=W0702
            mevis.MLAB.logError(f"Exception occurred in {funcName}:")
            printTestFunctionException()
 
        # Clean up the environment.
        self._testHelper.popChangeSet()
        while self._testHelper.getChangeSetStackLength() > changeSetLength:
            mevis.MLAB.logError("You should pop your ChangeSets by yourself!")
            self._testHelper.popChangeSet()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __stopCollectingAndStoreAsStartupLog ------------------------------------------------------------------------------{{{-
    
    def __stopCollectingAndStoreAsStartupLog(self):
        if MLABpriv.hadLogWarnings() or MLABpriv.hadLogErrors():
            self.__collectedStartupLog = MLABpriv.collectedLog()
            self.__collectedStartupErrorsState = MLABpriv.hadLogErrors()
            self.__collectedStartupWarningsState = MLABpriv.hadLogWarnings()
        else:
            self.__collectedStartupLog = None
            self.__collectedStartupErrorsState = False
            self.__collectedStartupWarningsState = False
        MLABpriv.stopLogCollect()
 
    # -- def _loadScriptFile --------------------------------------------------------------------{{{-
    
    def _loadScriptFile(self, filename):
        self._ctx = self._testHelper.getGlobalContext().addModule(filename)
        if self._ctx:
            # set dynamic property so that MeVisLab and MATE know that this is a TestCase
            self._ctx.setProperty("IsTestCase", True)
            self._ctx.setProperty("TestCaseName", self._testCaseName)
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def callFieldValueTestCase --------------------------------------------------------------{{{-
    
    def callFieldValueTestCase(self, filename, testcase):
        fieldValueTestCaseSet = FieldValueTestCaseSet(self._ctx)
        if not fieldValueTestCaseSet.load(filename):
            mevis.MLAB.logError(f"Failed ot load field-value test ({filename}).")
        else:
            testCase = fieldValueTestCaseSet.get(testcase)
            testCase.applyParameterization(TestHelper.getInstance().getChangeSet(), verbose=True)
            if not testCase.verifyExpectedResults(verbose=True):
                mevis.MLAB.logError("Failed to verify expected results.")
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __createNormalTestNode ----------------------------------------------------------------------{{{-
    
    def __createNormalTestNode(self, virtualFunctionName, functionName, functionBaseName, functionDict, isDisabled):
        retVal = True
        if not functionBaseName in functionDict["names"]:
            xmlNode = self.__getFunctionInfo(
                functionBaseName, functionName, isDisabled=isDisabled, sourceName=functionName
            )
            functionDict["names"].append(functionBaseName)
            functionDict["nodes"][virtualFunctionName] = xmlNode
        else:
            mevis.MLAB.logError(f"Function with name {functionBaseName} already defined!")
            retVal = False
        return retVal
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def _callGroupFunction(self, groupFunctionName):
        return self._ctx.call(groupFunctionName, [])
 
    # -- def __createTestGroupNode ---------------------------------------------------------------{{{-
    
    def __createTestGroupNode(self, groupFunctionName, orderString, groupName, functionDict, isDisabled):
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test group with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the field-value test case set file.
        functionList = self._callGroupFunction(groupFunctionName)
        if not functionList:
            mevis.MLAB.logError(f"Failed to build virtual functions for the {groupFunctionName} test.")
            return
 
        if type(functionList) not in (list, tuple):
            mevis.MLAB.logError(
                f"The test group function ({groupFunctionName}) must return a list or tuple with function objects!"
            )
            return
 
        virtualGroupName = f"TEST{orderString}_{groupName}"
        groupXMLNode = etree.Element(
            "GroupNode", name=virtualGroupName, basename=groupName, type="TestGroup", sourceName=groupFunctionName
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self._ctx.scriptVariable(f"{groupFunctionName}.__doc__")
 
        nameToFunctionMap = {}
        for function in functionList:
            # allow function names as string
            if isinstance(function, str):
                function = self._ctx.scriptVariable(function)
            nameToFunctionMap[function.__name__] = function
 
        for functionName in sorted(nameToFunctionMap.keys()):
            function = nameToFunctionMap[functionName]
 
            functionNodeName = functionName
            if functionNodeName.startswith("DISABLED_"):
                functionNodeName = functionNodeName[len("DISABLED_") :]
 
            if (
                functionNodeName not in functionDict["nodes"]
                or functionDict["nodes"][functionNodeName].tag != "Function"
            ):
                mevis.MLAB.logError(
                    f"Failed to compile function list. Given function {functionNodeName} not a test function "
                    f"or function already part of a test function group."
                )
                return False
 
            # Update function info, i.e. move the function node to the group node. The
            # dictionary with the virtual function names must be updated too.
            virtualFunctionName = (
                f"{virtualGroupName}_"
                f"{functionNodeName[4:] if not functionNodeName.startswith('DISABLED_') else functionNodeName[13:]}"
            )
 
            functionNode = functionDict["nodes"][functionNodeName]
            functionNode.set("name", virtualFunctionName)
            # mark function as disabled if the whole group is disabled; otherwise, keep the current state of the function
            # this way explicitly disabled functions stay disabled
            if isDisabled:
                functionNode.set("is_disabled", str(isDisabled))
 
            # Remove function from dictionary of nodes. It will be part of the group!
            del functionDict["nodes"][functionNodeName]
 
            # Remove old virtual function
            self.__functionDict.pop(functionName)
            self.__functionDict[virtualFunctionName] = functionName
 
            groupXMLNode.append(functionNode)
 
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualGroupName] = groupXMLNode
        return True
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __createFieldValueTestNode ----------------------------------------------------------{{{-
    
    def __createFieldValueTestNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
        # Find all the field-value test case functions, expand them and add the
        # single items to the virtual function list.
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the field-value test case set file.
        self._testHelper.pushChangeSet()
        retVal = self._ctx.call(functionName, [])
        self._testHelper.popChangeSet()
 
        filename, testCaseList = retVal if type(retVal) in (list, tuple) else (retVal, None)
 
        groupXMLNode = etree.Element(
            "GroupNode",
            name=virtualFunctionBaseName,
            basename=groupName,
            type="FieldValueTest",
            sourceName=functionName,
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
        # Keep in mind that duplicate names aren't allowed in field-value test case
        # sets either.
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
 
        fvtcs = FieldValueTestCaseSet(self._ctx)
        if not fvtcs.load(filename):
            mevis.MLAB.logError(f"Failed to load the field-value test case set ({filename})!")
            return False
 
        if not testCaseList:
            for testcase in sorted(fvtcs.getList()):
                virtualFunctionName = f"{virtualFunctionBaseName}_{testcase}"
                groupXMLNode.append(
                    self.__getFunctionInfo(
                        testcase,
                        virtualFunctionName,
                        (self.callFieldValueTestCasecallFieldValueTestCase, (filename, testcase)),
                        isDisabled=isDisabled,
                        sourceName=functionName,
                    )
                )
        else:
            availableTestCaseList = fvtcs.getList()
            for testcase in sorted(testCaseList):
                if testcase in availableTestCaseList:
                    virtualFunctionName = f"{virtualFunctionBaseName}_{testcase}"
                    groupXMLNode.append(
                        self.__getFunctionInfo(
                            testcase,
                            virtualFunctionName,
                            (self.callFieldValueTestCasecallFieldValueTestCase, (filename, testcase)),
                            isDisabled=isDisabled,
                            sourceName=functionName,
                        )
                    )
                else:
                    mevis.MLAB.logError(f"The field-value test case {testcase} is unknown!")
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __createIterativeTestNode -----------------------------------------------------------{{{-
    
    def __createIterativeTestNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
        # Find all iterative test functions, expand them and add the single items
        # to the virtual function list.
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the iterations and the actual function to call.
        self._testHelper.pushChangeSet()
        retVal = self._ctx.call(functionName, [])
        self._testHelper.popChangeSet()
 
        try:
            iterator, function = retVal
        except:  # pylint: disable=W0702
            mevis.MLAB.logError(
                f"The iterative test function ({functionName}) must return a list or dictionary"
                f" with parameters and a function to call!"
            )
            return
 
        groupXMLNode = etree.Element(
            "GroupNode", name=virtualFunctionBaseName, basename=groupName, type="IterativeTest", sourceName=functionName
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
        # Duplicate item nodes are not possible by design (either an index is used
        # or a dictionary).
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
        regExpFunctionNames = re.compile(r"^[.\w]+$")
 
        # Create virtual functions for given iterative test.
        if isinstance(iterator, (list, tuple)):
            ctr = 0
            # Let the index used to identify the function be zero padded.
            justificationWidth = len(unicode(len(iterator) - 1))
            for item in iterator:
                # The zero padded identifier.
                virtualFunctionId = unicode(ctr).rjust(justificationWidth, "0")
                args = [item] if (type(item) not in (list, tuple)) else item
                virtualFunctionName = f"{virtualFunctionBaseName}_{virtualFunctionId}"
                if regExpFunctionNames.match(virtualFunctionName):
                    groupXMLNode.append(
                        self.__getFunctionInfo(
                            virtualFunctionId,
                            virtualFunctionName,
                            (function, args),
                            isDisabled=isDisabled,
                            sourceName=functionName,
                        )
                    )
                else:
                    mevis.MLAB.logError(
                        f"The iterative test function ({functionName}) defines non-alphanumeric virtual "
                        f"function name {virtualFunctionName}!"
                    )
                ctr += 1
        elif isinstance(iterator, dict):
            for item in sorted(iterator.keys()):
                args = [iterator[item]] if (type(iterator[item]) not in (list, tuple)) else iterator[item]
                virtualFunctionName = f"{virtualFunctionBaseName}_{item}"
                if regExpFunctionNames.match(virtualFunctionName):
                    groupXMLNode.append(
                        self.__getFunctionInfo(
                            unicode(item),
                            virtualFunctionName,
                            (function, args),
                            isDisabled=isDisabled,
                            sourceName=functionName,
                        )
                    )
                else:
                    mevis.MLAB.logError(
                        f"The iterative test function ({functionName}) defines non-alphanumeric virtual "
                        f"function name {virtualFunctionName}!"
                    )
        else:
            mevis.MLAB.logError(
                f"The iterative test function ({functionName}) must return a list or dictionary with parameters!"
            )
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __createUnitTestWrapperNode -----------------------------------------------------------{{{-
    
    def __createUnitTestWrapperNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
        # Find all unit test wrapper functions, expand them and add the single items
        # to the virtual function list.
 
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the unit test and the actual function to call.
        self._testHelper.pushChangeSet()
        suite = self._ctx.call(functionName, [])
        self._testHelper.popChangeSet()
 
        if not isinstance(suite, unittest.TestSuite):
            mevis.MLAB.logError(f"The unit test wrapper function ({functionName}) must return a unittest.TestSuite!")
            return
 
        groupXMLNode = etree.Element(
            "GroupNode",
            name=virtualFunctionBaseName,
            basename=groupName,
            type="UnitTestWrapper",
            sourceName=functionName,
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
        # Duplicate item nodes are not possible by design (either an index is used
        # or a dictionary).
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
 
        def _isSkipped(test):
            isClassSkipped = getattr(test.__class__, "__unittest_skip__", False)
            testMethod = getattr(test, test._testMethodName)  # pylint: disable=W0212
            isMethodSkipped = getattr(testMethod, "__unittest_skip__", False)
            return isClassSkipped or isMethodSkipped
 
        # Create virtual functions for given unit test wrapper
        def _unpackSuite(suite):
            allTests = []
            for test in suite._tests:  # pylint: disable=W0212
                if isinstance(test, unittest.TestSuite):
                    allTests.extend(_unpackSuite(test))
                else:
                    if not _isSkipped(test):
                        allTests.append(test)
            return allTests
 
        allTests = _unpackSuite(suite)
 
        def _makeName(test):
            testCaseName = test.__class__.__name__
            testFunctionName = test._testMethodName  # pylint: disable=W0212
            return "{0}_{1}".format(testCaseName, testFunctionName)
 
        allTests.sort(key=_makeName)
        testNames = list(map(_makeName, allTests))
 
        # It could be smarter to integrate this more deeply into the callTestFunction part.
        def _runUnitTestFunction(testFunction):
            testResult = UnitTestResult()
            testResult._previousTestClass = self.__previousTestClass
            if self.__dummyTestSuite is None:
                self.__dummyTestSuite = unittest.TestSuite()
            # handle setup of class and module fixtures (copied from unittest.TestSuite):
            self.__dummyTestSuite._tearDownPreviousClass(testFunction, testResult)
            self.__dummyTestSuite._handleModuleFixture(testFunction, testResult)
            self.__dummyTestSuite._handleClassSetUp(testFunction, testResult)
            self.__previousTestClass = testFunction.__class__
            # run test:
            testFunction.run(testResult)
            testResult.logResult("Test was successful")
 
        for testName, testFunction in zip(testNames, allTests):
            testMethod = getattr(testFunction, testFunction._testMethodName)
            funcXMLNode = self.__getFunctionInfo(
                testName,
                "{}_{}".format(virtualFunctionBaseName, testName),
                (_runUnitTestFunction, [testFunction]),
                isDisabled=isDisabled,
            )
            if hasattr(testMethod, "__func__"):  # testMethod is really a method
                funcXMLNode.attrib["sourceFilename"] = testMethod.__func__.__code__.co_filename
                funcXMLNode.attrib["sourceLine"] = str(testMethod.__func__.__code__.co_firstlineno)
            elif hasattr(testMethod, "__code__"):  # testMethod is actually a function
                funcXMLNode.attrib["sourceFilename"] = testMethod.__code__.co_filename
                funcXMLNode.attrib["sourceLine"] = str(testMethod.__code__.co_firstlineno)
            else:  # None or unsupported type - should we raise here?
                funcXMLNode.attrib["sourceFilename"] = ""
                funcXMLNode.attrib["sourceLine"] = "0"
            groupXMLNode.append(funcXMLNode)
 
    def _callableFunctions(self):
        return self._ctx.callableFunctions()
 
    # -- def __buildVirtualFunctions ---------------------------------------------------------------------{{{-
    
    def _buildVirtualFunctions(self, isNewlyCreated):
        # A list containing the found function names (to prevent duplicates).
        functionInformationDict = {"names": [], "nodes": {}}
 
        regExpDict = {
            "TestFunction": re.compile(r"^(DISABLED_)?TEST(\d*)_(\w+)$"),
            "TestGroup": re.compile(r"^(DISABLED_)?GROUP(\d*)_(\w+)$"),
            "FieldValueTest": re.compile(r"^(DISABLED_)?FIELDVALUETEST(\d*)_(\w+)$"),
            "IterativeTest": re.compile(r"^(DISABLED_)?ITERATIVETEST(\d*)_(\w+)$"),
            "UnitTestWrapper": re.compile(r"^(DISABLED_)?UNITTEST(\d*)_(\w+)$"),
        }
 
        # The result node.
        functionsNode = etree.SubElement(self.__xmlRoot, "TestFunctions")
 
        # Check the optional list of disabled test functions (it is defined by the disabledTestFunction() decorator)
        disabledFunctions = self._ctx.scriptVariable("MLABTC_DISABLED_TEST_FUNCTIONS") or []
 
        # The dictionary of all test functions sorted by type.
        functionDict = {
            "TestFunction": [],
            "TestGroup": [],
            "FieldValueTest": [],
            "IterativeTest": [],
            "UnitTestWrapper": [],
        }
        for functionName in self._callableFunctions():
            for prefix, functionList in (
                ("TEST", functionDict["TestFunction"]),
                ("GROUP", functionDict["TestGroup"]),
                ("FIELDVALUETEST", functionDict["FieldValueTest"]),
                ("ITERATIVETEST", functionDict["IterativeTest"]),
                ("UNITTEST", functionDict["UnitTestWrapper"]),
            ):
                if functionName.startswith(prefix) or functionName.startswith("DISABLED_" + prefix):
                    functionList.append(functionName)
 
        for functionName in sorted(functionDict["TestFunction"]):
            mObj = regExpDict["TestFunction"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of test function invalid ({functionName})")
            else:
                disabledString, orderString, functionBaseName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{functionBaseName}"
                self.__createNormalTestNode(
                    fname, functionName, functionBaseName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["TestGroup"]):
            mObj = regExpDict["TestGroup"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of test group invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                self.__createTestGroupNode(
                    functionName, orderString, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["FieldValueTest"]):
            mObj = regExpDict["FieldValueTest"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of field value test invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{groupName}"
                self.__createFieldValueTestNode(
                    fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["IterativeTest"]):
            mObj = regExpDict["IterativeTest"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of iterative test invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{groupName}"
                self.__createIterativeTestNode(
                    fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["UnitTestWrapper"]):
            mObj = regExpDict["UnitTestWrapper"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of unit test wrapper invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{groupName}"
                self.__createUnitTestWrapperNode(
                    fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        # Append nodes in correct order. Essential for other methods to work
        # properly.
        keys = sorted(functionInformationDict["nodes"].keys())
        if len(keys) > 0:
            for functionName in keys:
                functionsNode.append(functionInformationDict["nodes"][functionName])
        elif not isNewlyCreated:
            mevis.MLAB.logError(f"No test functions found in TestCase {self._testCaseName}!")
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __getFunctionInfo -------------------------------------------------------------------{{{-
    
    def __getFunctionInfo(self, functionBaseName, functionName, calledFunction=None, isDisabled=False, sourceName=""):
        self.__functionDict.setdefault(functionName, calledFunction)
        functionXMLNode = etree.Element(
            "Function", basename=functionBaseName, name=functionName, sourceName=sourceName, is_disabled=str(isDisabled)
        )
        documentation = self.getDocumentation(functionName)
        etree.SubElement(functionXMLNode, "Documentation").text = documentation if documentation else ""
        return functionXMLNode
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def isValid -----------------------------------------------------------------------------{{{-
    
    def isValid(self):
        return self.__valid
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def hasNetwork --------------------------------------------------------------------------{{{-
    
    def hasNetwork(self):
        return os.path.exists(self.networkFileName())
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def networkFileName --------------------------------------------------------------------------{{{-
    
    def networkFileName(self):
        return self._ctx.network().filename()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def openFiles ---------------------------------------------------------------------------{{{-
    
    def openFiles(self):
        typeName = self.getName()
        for fileName in self._ctx.getScriptSourceFiles():
            mevis.MLAB.priv().openRelatedFile(typeName, fileName)
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def openNetwork -------------------------------------------------------------------------{{{-
    
    def openNetwork(self):
        if self._ctx:
            mevis.MLAB.priv().showIDE()
            self._ctx.showInternalNetwork()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getType -----------------------------------------------------------------------------{{{-
    
    def getType(self):
        return self._testCaseType
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getModule -----------------------------------------------------------------------------{{{-
    
    def getModule(self):
        return self._ctx
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getName -----------------------------------------------------------------------------{{{-
    
    def getName(self):
        return self._testCaseName
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getListOfTestFuncs ------------------------------------------------------------------{{{-
    
    def getListOfTestFuncs(self):
        return sorted(self.__functionDict.keys())
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getPackage --------------------------------------------------------------------------{{{-
    
    def getPackage(self):
        return self.__xmlRoot.find("Package").text
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getTestCaseNode ---------------------------------------------------------------------{{{-
    
    def getTestCaseNode(self):
        return self.__xmlRoot
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __callSetUpTestCase -----------------------------------------------------------------{{{-
    
    def __callSetUpTestCase(self):
        if "setUpTestCase" in self._ctx.callableFunctions():
            self._ctx.call("setUpTestCase", [])
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __callTearDownTestCase --------------------------------------------------------------{{{-
    
    def __callTearDownTestCase(self):
        if "tearDownTestCase" in self._ctx.callableFunctions():
            self._ctx.call("tearDownTestCase", [])
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def __tearDown(self):
        if self.__setupTestCaseCalled:
            mevis.MLAB.log("Calling tearDown() function...")
            self.__callTearDownTestCase()
        if self.__dummyTestSuite:
            # unittest support: clean up after running unittest.TestCases
            testResult = UnitTestResult()
            testResult._previousTestClass = self.__previousTestClass
            self.__dummyTestSuite._tearDownPreviousClass(None, testResult)
            self.__dummyTestSuite._handleModuleTearDown(testResult)
            testResult.logResult("")
            self.__previousTestClass = None
            self.__dummyTestSuite = None
        if TestSupport.Logging.gStopped:
            raise CancelTestException
        else:
            mevis.MLAB.log("Destroying test case context...")
            self.destroyContext()
 
    # -- def run ---------------------------------------------------------------------------------{{{-
    
    def run(self, resultNode):
 
        self.activate()
 
        TestSupport.Logging.gStopped = False
 
        logHandler = self._testHelper.getLogHandler()
 
        availTestFunctions = self.getListOfTestFuncs()
 
        # Build dict mapping test function names to xml nodes.
        funcList = resultNode.findall(".//Function")
        length = len(funcList)
 
        self.__currentResultNode = resultNode
 
        # Make sure the virtual test function for loading the test case is the first, if it exists.
        # This ensures that initial errors/warnings are appended to this function. Accordingly
        # the unload test case function must be last.
        loadTestCaseFunctionHandled = False
        unloadTestCaseFunctionHandled = False
        for i in range(0, length):
            funcNode = funcList[i]
            funcName = funcNode.get("name")
            if not loadTestCaseFunctionHandled and funcName == Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION:
                del funcList[i]
                funcList.insert(0, funcNode)
                loadTestCaseFunctionHandled = True
            elif not unloadTestCaseFunctionHandled and funcName == Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION:
                del funcList[i]
                funcList.append(funcNode)
                unloadTestCaseFunctionHandled = True
            if loadTestCaseFunctionHandled and unloadTestCaseFunctionHandled:
                break
 
        self.__setupTestCaseCalled = False
 
        # Iterate over all test functions (sorted by name) that should be tested.
        # The iteration with an index is required to call setUpTestCase and
        # tearDownTestCase properly.
        for i in range(0, length):
            funcNode = funcList[i]
            funcName = funcNode.get("name")
 
            isLastTestFunction = i == length - 1
            data = RunTestFunctionData(funcNode, funcName, logHandler, isLastTestFunction)
 
            if not self.__prepareRunTestFunction(data, availTestFunctions):
                continue
 
            self.__runTestFunction(data)
            self.__finishRunTestFunction(data)
            if data.cancelled:
                # Do NOT call __tearDown() when 'Stop on error' flag is set and an error occured.
                if not (TestSupport.Logging.gStopOnFirstError and logHandler.numErrors() > 0):
                    self.__tearDown()
                break
 
        self.deactivate()
 
    def __prepareRunTestFunction(self, data, availTestFunctions):
        if self.__perFunctionResultDirectory:
            funcNameForResultDir = data.funcName
        else:
            funcNameForResultDir = None
        self._testHelper.setTestCaseResultDirectory(self.getName().replace("::", os.sep), funcNameForResultDir)
 
        shouldRunTestFunction = True
 
        # Verify that a function with the given name exists.
        if data.funcName not in availTestFunctions and data.funcName not in Utils.VIRTUAL_TEST_CASE_FUNCTIONS:
            mevis.MLAB.logError(f"Function {data.funcName} does not exist in test case {self.getName()}!")
            status = Utils.TEST_STATUS_FUNCTION_NOT_FOUND
            data.funcNode.set("status", str(status))
            shouldRunTestFunction = False
        return shouldRunTestFunction
 
    def __runTestFunction(self, data):
        data.started = time.time()
 
        try:
            data.logHandler.startTestFunction(self.getType(), self.getName(), data.funcName)
            if self.__testCaseListener:
                self.__testCaseListener.startTestCaseFunction(self.getType(), self.getName(), data.funcName)
 
            MLABpriv.clearLogState()
 
            # append earlier testcase error/warnings to the first test function that is executed
            self.__appendCollectedStartupLog(data.logHandler)
 
            self.__setupTestCaseAndRunTestFunction(data)
 
            if data.funcName == Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION:
                self.__tearDown()
            if TestSupport.Logging.gStopped:
                raise CancelTestException
        except CancelTestException:
            data.cancelled = True
 
    def __finishRunTestFunction(self, data):
        for extraTestCaseResult in self._testHelper.takeExtraTestCaseResults():
            self.addExtraTestCaseResult(extraTestCaseResult)
 
        # process events so that any delayed messages are also part of the test function
        mevis.MLAB.processEvents()
        # force messages from other threads to be printed inside of this test function execution
        MLABpriv.flushPendingMessages()
 
        duration = time.time() - data.started
 
        # Gather status and additional information.
        status = Utils.TEST_STATUS_OK
        if MLABpriv.hadLogWarnings() or self.__collectedStartupWarningsState:
            status = Utils.TEST_STATUS_WARNING
        if MLABpriv.hadLogErrors() or self.__collectedStartupErrorsState:
            status = Utils.TEST_STATUS_ERROR
        self.__collectedStartupErrorsState = False
        self.__collectedStartupWarningsState = False
        data.logHandler.stopTestFunction(status, duration)
        if self.__testCaseListener:
            self.__testCaseListener.endTestCaseFunction(status, duration)
 
        if data.cancelled:
            status = Utils.TEST_STATUS_CANCELLED
        data.funcNode.set("status", str(status))
        data.funcNode.set("duration", str(duration))
        data.funcNode.append(data.logHandler.getMessages())
 
    def __appendCollectedStartupLog(self, logHandler):
        if self.__collectedStartupLog:
            log = self.__collectedStartupLog
            msgs = log.split("<br>")
            if len(msgs):
                mevis.MLAB.log("TestCase setup errors/warnings:")
                for msg in msgs:
                    if msg != "":
                        logHandler.appendMessage(msg)
                mevis.MLAB.log("TestCase execution:")
            self.__collectedStartupLog = None
 
    def __setupTestCaseAndRunTestFunction(self, data):
        if data.funcNode.get("is_disabled", str(False)) == str(False):
            if not self.__setupTestCaseCalled:
                self.__setupTestCaseCalled = True
                self.__callSetUpTestCase()
                if TestSupport.Logging.gStopped:
                    raise CancelTestException
            mevis.MLAB.log(f"Calling test function {data.funcName}()")
            self.__callTestFunction(data.funcName)
        else:
            mevis.MLAB.log(f"Skipping disabled test function {data.funcName}()")
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def getDocumentation --------------------------------------------------------------------{{{-
    
    def getDocumentation(self, functionName):
        docu = self._ctx.scriptVariable(f"{functionName}.__doc__")
        testDocu = docu if docu else ""
        return re.sub(r"(?m)^\s+", "", testDocu)
 
    # --------------------------------------------------------------------------------------------}}}-
 
 
# ----------------------------------------------------------------------------------------------}}}-