TestCase.py

Go to the documentation of this file.

#
# Copyright 2009, MeVis Medical Solutions AG
#
# The user may use this file in accordance with the license agreement provided with
# the Software or, alternatively, in accordance with the terms contained in a
# written agreement between the user and MeVis Medical Solutions AG.
#
# For further information use the contact form at https://www.mevislab.de/contact
#
import contextlib
 
 
 
# -- system imports ----------------------------------------------------------------------------{{{-
import inspect
import os
import re
import sys
import time
import traceback
import unittest
import asyncio
from PythonQt import QtCore
 
import xml.etree.cElementTree as etree
 
# ----------------------------------------------------------------------------------------------}}}-
 
# -- local imports -----------------------------------------------------------------------------{{{-
import mevis
from TestSupport.FieldValueTests import FieldValueTestCaseSet
 
from TestSupport.TestHelper import TestHelper, CancelTestException
from . import Utils
import TestSupport.Logging
import TestSupport.Macros
 
# ----------------------------------------------------------------------------------------------}}}-
 
# -- local imports -----------------------------------------------------------------------------{{{-
MLABpriv = mevis.MLAB.priv()
# ----------------------------------------------------------------------------------------------}}}-
 
 
def printTestFunctionException():
    """Prints the exception that occurred during a test function."""
    exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
    tb = exceptionTraceback.tb_next
    traceback.print_tb(tb, limit=None, file=sys.stderr)
    lines = traceback.format_exception_only(exceptionType, exceptionValue)
    for line in lines:
        sys.stderr.write(line)
 
 
class RunTestFunctionData:
    def __init__(self, funcNode, funcName, logHandler, isLastTestFunction):
        self.funcNode = funcNode
        self.funcName = funcName
        self.logHandler = logHandler
        self.isLastTestFunction = isLastTestFunction
        self.started = None
        self.cancelled = False
 
 
class LogCollectorBase:
    def __init__(self, testHelper):
        self.hasWarnings = False
        self.hasErrors = False
        self._testHelper = testHelper
 
    @property
    def logHandler(self):
        return self._testHelper.getLogHandler() if self._testHelper else None
 
    @contextlib.contextmanager
    def collect(self):
        try:
            self.startLogCollect()
            yield
        finally:
            self.stopLogCollect()
 
    def startLogCollect(self):
        if self.logHandler:
            MLABpriv.aboutToLogMessage.connect(self.logHandler.aboutToLogMessage)
        MLABpriv.clearLogState()
        MLABpriv.startLogCollect()
 
    def stopLogCollect(self):
        if self.logHandler:
            MLABpriv.aboutToLogMessage.disconnect(self.logHandler.aboutToLogMessage)
        self.hasWarnings = self.hasWarnings or MLABpriv.hadLogWarnings()
        self.hasErrors = self.hasErrors or MLABpriv.hadLogErrors()
        MLABpriv.clearLogState()
        MLABpriv.stopLogCollect()
 
    def clearState(self):
        self.hasWarnings = False
        self.hasErrors = False
 
 
class LogCollector(LogCollectorBase):
    """Allows collecting log messages and logging them later.
    Keeps the logging state during log collection."""
 
    def __init__(self, testHelper=None):
        # the testHelper is set to None in the load phase, because the log handler is not available yet,
        # and we don't need it as we won't use test functions like "hasErrors" here
        super().__init__(testHelper)
        self.messageCache = []
        self.log = ""
 
    def startLogCollect(self):
        if self.logHandler:
            MLABpriv.loggingMessage.connect(
                lambda msg: self.logHandler.appendMessage(msg, messageCache=self.messageCache)
            )
        super().startLogCollect()
 
    def stopLogCollect(self):
        super().stopLogCollect()
        if self.logHandler:
            MLABpriv.loggingMessage.disconnect()
        self.log = MLABpriv.collectedLog()
        MLABpriv.clearLogCollect()
 
    def logMessages(self, logHandler):
        if self.logHandler:
            logHandler.flushMessageCache(self.messageCache)
            self.messageCache = []
        else:
            for log in self.log.split("<br>"):
                if log:
                    logHandler.appendMessage(log)
            self.log = ""
 
 
class TestLogger(LogCollectorBase):
    """Allows logging messages via a given log handler and records the logging state."""
 
    def startLogCollect(self):
        MLABpriv.loggingMessage.connect(self.logHandler.appendMessage)
        super().startLogCollect()
 
    def stopLogCollect(self):
        super().stopLogCollect()
        MLABpriv.loggingMessage.disconnect(self.logHandler.appendMessage)
 
 
class UnitTestResult(unittest.TestResult):
 
    # deactivate stdout/stderr redirection, the output is captured by the TestCenter anyway
    def _setupStdout(self):
        pass
 
    def _restoreStdout(self):
        pass
 
    def logResult(self, successMsg):
        if self.wasSuccessful() and successMsg:
            mevis.MLAB.log(successMsg)
        if self.errors:
            for error in self.errors:
                mevis.MLAB.logError(error[1])
        if self.failures:
            for failure in self.failures:
                mevis.MLAB.logError(failure[1])
 
 
class TestCase:
    """The superclass for the different test case categories."""
 
    def __init__(self, testCaseName, isNewlyCreated=False):
        """Initialize the test case.
 
        @param testCaseName The name of the test case.
        @param isNewlyCreated Indicates if the test case is newly created to suppress the warning
               that no test functions exist.
        """
        self._ctx = None  # testcase context
        self._testCaseName = testCaseName
        self._testHelper = TestHelper.getInstance()
        self.__testCaseListener = None
 
        self.__functionDict = {}
 
        
        self.__dummyTestSuite = None
 
        
        self.__previousTestClass = None
        # XML node for extra test case results
        self.__extraTestCasesResults = None
        # The current result node.
        self.__currentResultNode = None
 
        self.__hasSetUpError = False
        self.__setUpTestCaseCalled = False
 
        # we have to collect the logs in the setup phase to
        # avoid mixing load and setup phase logs
        self.__loadLogCollector = LogCollector(self._testHelper)
        self.__setupLogCollector = LogCollector(self._testHelper)
        # during tests and teardown phase log messages can be directly sent to the log
        self.__testLogger = TestLogger(self._testHelper)
 
        with self.__loadLogCollector.collect():
            infoDict = self._getTestCaseInfo(testCaseName)
            associatedModules = infoDict.get("associatedModules", None)
            if associatedModules is not None:
                deprecatedModules = 0
                for associatedModule in associatedModules:
                    moduleInfo = mevis.MLAB.moduleInfo(associatedModule)
                    if "deprecated" in moduleInfo.get("group", "").lower():
                        deprecatedModules += 1
                if deprecatedModules == len(associatedModules):
                    MLABpriv.ignoreDeprecationWarnings(associatedModules)
 
            # Get the basic test case information and store them in the XML node.
            self.__xmlRoot = etree.Element("TestCase", name=self._testCaseName, type=self.getType())
            self.__xmlRoot.set("timeout", infoDict.get("timeout", "0"))
            self.__perFunctionResultDirectory = mevis.MLAB.valueIsTrue(infoDict.get("perFunctionResultDirectory", "1"))
 
            etree.SubElement(self.__xmlRoot, "Package").text = infoDict["package"]
            etree.SubElement(self.__xmlRoot, "Author").text = infoDict["author"]
            etree.SubElement(self.__xmlRoot, "Maintainer").text = infoDict.get("maintainer", "")
            etree.SubElement(self.__xmlRoot, "Comment").text = infoDict.get("comment", "")
            etree.SubElement(self.__xmlRoot, "File").text = infoDict["file"]
            etree.SubElement(self.__xmlRoot, "Line").text = str(infoDict["lineno"])
            showTestFunctionSortLiterals = "0"
            if "showTestFunctionSortingLiterals" in infoDict:
                shouldShowLiterals = mevis.MLAB.valueIsTrue(infoDict["showTestFunctionSortingLiterals"])
                showTestFunctionSortLiterals = "1" if shouldShowLiterals else "0"
            etree.SubElement(self.__xmlRoot, "showTestFunctionSortingLiterals").text = showTestFunctionSortLiterals
            if "preferredRenderer" in infoDict:
                etree.SubElement(self.__xmlRoot, "preferredRenderer").text = infoDict["preferredRenderer"]
 
            # Set the data directory. If not specified use default value.
            if "dataDirectory" in infoDict:
                self._dataDirectory = infoDict["dataDirectory"]
            else:
                self._dataDirectory = os.path.join(os.path.dirname(infoDict["file"]), "Data")
 
            if not (infoDict and "scriptFile" in infoDict):
                mevis.MLAB.logError(f"Missing scriptFile tag in TestCase {testCaseName}.")
            else:
                try:
                    self._loadScriptFile(infoDict["scriptFile"])
                    # we need to activate the TestCase, because building of the virtual functions will call into
                    # TestHelper functions...
                    self.activate()
 
                    self._buildVirtualFunctions(isNewlyCreated)
 
                    self.deactivate()
 
                    etree.SubElement(self.__xmlRoot, "Documentation").text = self._ctx.scriptVariable("__doc__")
                except Exception as e:
                    mevis.MLAB.logError(f"Failed to load script file {infoDict['scriptFile']}. No tests created.")
                    raise
 
            if "setUpTestCase" in self._ctx.callableFunctions():
                # create a virtual test function to show the setup messages
                self.__createVirtualTestFunction(
                    "SetUpTestCase",
                    Utils.SETUP_TEST_CASE_FUNCTION,
                    "Collects messages while the test case is set up",
                    prepend=True,
                )
 
            if "tearDownTestCase" in self._ctx.callableFunctions():
                # create a virtual test function to show messages from the test case tear down
                self.__createVirtualTestFunction(
                    "TearDownTestCase",
                    Utils.TEARDOWN_TEST_CASE_FUNCTION,
                    "Collects messages during the teardown phase of the test case",
                    prepend=False,
                )
 
            # create a virtual test function to show unload errors
            self.__createVirtualTestFunction(
                "UnloadTestCase",
                Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION,
                "Collects errors while the test case is unloaded",
                prepend=False,
            )
 
        MLABpriv.ignoreDeprecationWarnings()
        self.__valid = not self.__loadLogCollector.hasErrors
        if not self.__valid or self.__loadLogCollector.hasWarnings:
            # create a virtual test function to show errors and warnings during test case load
            self.__createVirtualTestFunction(
                "LoadTestCase",
                Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION,
                "Collects errors while the test case is loaded",
                prepend=True,
            )
 
    def __createVirtualTestFunction(self, basename, name, documentation, prepend):
        functionsNode = self.__xmlRoot.find("TestFunctions")
        if functionsNode is None:
            functionsNode = etree.SubElement(self.__xmlRoot, "TestFunctions")
        functionXMLNode = functionsNode.makeelement(
            "Function", {"basename": basename, "name": name, "is_disabled": "False"}
        )
        if prepend:
            functionsNode.insert(0, functionXMLNode)
        else:
            functionsNode.append(functionXMLNode)
        etree.SubElement(functionXMLNode, "Documentation").text = documentation
 
    def _getTestCaseInfo(self, testCaseName):
        return mevis.MLABTestCaseDatabase.testCaseInfo(testCaseName)
 
    def __del__(self):
        self.destroyContext()
 
    def destroyContext(self):
        if self._ctx:
            try:
                self._ctx.remove()
            finally:
                self._ctx = None
 
    def activate(self):
        """Registers the test case at the TestHelper. Does not support recursive calls."""
        # Tell the TestHelper about us, but don't set a reference
        # to self to the TestHelper to avoid that it reference counts us
        self._testHelper.setMacrosLogOnSuccess(True)
        self._testHelper.setTestCaseContext(self._ctx)
        self._testHelper.setTestCaseName(self._testCaseName)
        self._testHelper.setTestCaseDataDirectory(self._dataDirectory)
        self._testHelper.pushChangeSet()
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def deactivate(self):
        """Resets the test helper. Does not support recursive calls."""
        self._testHelper.setTestCaseContext(None)
        self._testHelper.setTestCaseName("")
        self._testHelper.setTestCaseDataDirectory("")
        self._testHelper.popChangeSet()
        self._testHelper.setMacrosLogOnSuccess(True)
 
    # --------------------------------------------------------------------------------------------}}}-
 
    def setTestCaseListener(self, testCaseListener):
        self.__testCaseListener = testCaseListener
 
    # -- def addExtraTestCaseResult ------------------------------------------------------------------{{{-
    def addExtraTestCaseResult(self, extraTestCaseResult):
        assert self.__currentResultNode is not None
        assert self.__currentResultNode != self.__xmlRoot
        if self.__extraTestCasesResults is None:
            self.__extraTestCasesResults = etree.SubElement(self.__currentResultNode, "ExtraTestCasesResults")
        self.__extraTestCasesResults.append(extraTestCaseResult.toXML())
 
    def _getTestFunction(self, funcName):
        return self._ctx.scriptVariable(funcName)
 
    def __callTestFunction(self, funcName):
        """Call the test function with the given name.
 
        @param funcName The name of the function to execute.
        """
        if Utils.isInternalTestCaseFunction(funcName):
            return
 
        if not funcName in self.__functionDict:
            raise Exception("Unknown function!")
 
        changeSetLength = self._testHelper.pushChangeSet()
 
        callInfo = self.__functionDict[funcName]
 
        try:
            if callInfo:
                if type(callInfo) in (list, tuple):
                    testFunction, testArguments = callInfo
                    callWithKeywordArguments = False
                    if len(testArguments) == 1 and isinstance(testArguments[0], dict):
                        signature = inspect.signature(testFunction)
                        try:
                            # try to bind the test arguments as keyword arguments to the test function
                            signature.bind(**testArguments[0])
                            callWithKeywordArguments = True
                        except TypeError:
                            pass
                    if callWithKeywordArguments:
                        result = testFunction(**testArguments[0])
                    else:
                        result = testFunction(*testArguments)
                else:
                    result = self._getTestFunction(callInfo)()
            else:
                result = self._getTestFunction(funcName)()
 
            # if the result is a coroutine, we schedule it and process events until it
            # has finished... This allows test functions to use the async keyword.
            if asyncio.iscoroutine(result):
                self.futureDone = False
 
                def futureDone(*args):
                    self.futureDone = True
 
                future = asyncio.ensure_future(result)
                future.add_done_callback(futureDone)
                while not self.futureDone:
                    QtCore.QCoreApplication.processEvents()
                future.result()
 
            if TestSupport.Logging.gStopped:
                raise CancelTestException
 
        except CancelTestException:
            self._testHelper.clearChangeSet()
            raise
        except TestSupport.Macros.AssertException:
            # an assertion has failed and we should stop execution,
            # but this is an expected exception, so we don't print it.
            pass
        except:  # pylint: disable=W0702
            self._testHelper.clearChangeSet()
            mevis.MLAB.logError(f"Exception occurred in {funcName}:")
            printTestFunctionException()
 
        # Clean up the environment.
        self._testHelper.popChangeSet()
        while self._testHelper.getChangeSetStackLength() > changeSetLength:
            mevis.MLAB.logError("You should pop your ChangeSets by yourself!")
            self._testHelper.popChangeSet()
 
    def _loadScriptFile(self, filename):
        """Load the test cases script file.
 
        @param filename Name of the script file to be loaded.
        """
        self._ctx = self._testHelper.getGlobalContext().addModule(filename)
        if self._ctx:
            # set dynamic property so that MeVisLab and MATE know that this is a TestCase
            self._ctx.setProperty("IsTestCase", True)
            self._ctx.setProperty("TestCaseName", self._testCaseName)
 
    def callFieldValueTestCase(self, filename, testcase):
        """Call the given field-value test case from the given field-value test case set.
        This method is required as the FIELDVALUETEST test methods don't have a
        valid counterpart in the script file itself.
 
        @param filename The field-value test case set.
        @param testcase The test case to be executed.
        """
        fieldValueTestCaseSet = FieldValueTestCaseSet(self._ctx)
        if not fieldValueTestCaseSet.load(filename):
            mevis.MLAB.logError(f"Failed ot load field-value test ({filename}).")
        else:
            testCase = fieldValueTestCaseSet.get(testcase)
            testCase.applyParameterization(TestHelper.getInstance().getChangeSet(), verbose=True)
            if not testCase.verifyExpectedResults(verbose=True):
                mevis.MLAB.logError("Failed to verify expected results.")
 
    def __createNormalTestNode(self, virtualFunctionName, functionName, functionBaseName, functionDict, isDisabled):
        """Create the XML node for a normal test method.
 
        @param functionName The real name of the function.
        @param functionBaseName The base name (without the TEST_).
        @param functionDict The dictionary used to prevent duplicated function names.
        """
        retVal = True
        if not functionBaseName in functionDict["names"]:
            xmlNode = self.__getFunctionInfo(
                functionBaseName, functionName, isDisabled=isDisabled, sourceName=functionName
            )
            functionDict["names"].append(functionBaseName)
            functionDict["nodes"][virtualFunctionName] = xmlNode
        else:
            mevis.MLAB.logError(f"Function with name {functionBaseName} already defined!")
            retVal = False
        return retVal
 
    def _callGroupFunction(self, groupFunctionName):
        return self._ctx.call(groupFunctionName, [])
 
    def __createTestGroupNode(self, groupFunctionName, orderString, groupName, functionDict, isDisabled):
        """Create the XML information for an item in a test group.
 
        @param groupFunctionName The name of the group function.
        @param orderString The string used to determine the order of test functions.
        @param groupName The name of the group.
        @param functionDict The dictionary used to prevent duplicated function names.
        """
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test group with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the field-value test case set file.
        functionList = self._callGroupFunction(groupFunctionName)
        if not functionList:
            mevis.MLAB.logError(f"Failed to build virtual functions for the {groupFunctionName} test.")
            return
 
        if type(functionList) not in (list, tuple):
            mevis.MLAB.logError(
                f"The test group function ({groupFunctionName}) must return a list or tuple with function objects!"
            )
            return
 
        virtualGroupName = f"TEST{orderString}_{groupName}"
        groupXMLNode = etree.Element(
            "GroupNode", name=virtualGroupName, basename=groupName, type="TestGroup", sourceName=groupFunctionName
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self._ctx.scriptVariable(f"{groupFunctionName}.__doc__")
 
        nameToFunctionMap = {}
        for function in functionList:
            # allow function names as string
            if isinstance(function, str):
                function = self._ctx.scriptVariable(function)
            nameToFunctionMap[function.__name__] = function
 
        for functionName in sorted(nameToFunctionMap.keys()):
            function = nameToFunctionMap[functionName]
 
            functionNodeName = functionName
            if functionNodeName.startswith("DISABLED_"):
                functionNodeName = functionNodeName[len("DISABLED_") :]
 
            if (
                functionNodeName not in functionDict["nodes"]
                or functionDict["nodes"][functionNodeName].tag != "Function"
            ):
                mevis.MLAB.logError(
                    f"Failed to compile function list. Given function {functionNodeName} not a test function "
                    f"or function already part of a test function group."
                )
                return False
 
            # Update function info, i.e. move the function node to the group node. The
            # dictionary with the virtual function names must be updated too.
            virtualFunctionName = (
                f"{virtualGroupName}_"
                f"{functionNodeName[4:] if not functionNodeName.startswith('DISABLED_') else functionNodeName[13:]}"
            )
 
            functionNode = functionDict["nodes"][functionNodeName]
            functionNode.set("name", virtualFunctionName)
            # mark function as disabled if the whole group is disabled; otherwise, keep the current state of the function
            # this way explicitly disabled functions stay disabled
            if isDisabled:
                functionNode.set("is_disabled", str(isDisabled))
 
            # Remove function from dictionary of nodes. It will be part of the group!
            del functionDict["nodes"][functionNodeName]
 
            # Remove old virtual function
            self.__functionDict.pop(functionName)
            self.__functionDict[virtualFunctionName] = functionName
 
            groupXMLNode.append(functionNode)
 
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualGroupName] = groupXMLNode
        return True
 
    def __createFieldValueTestNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
        """Create the XML information for the given field-value test.
 
        @param virtualFunctionBaseName The base name of the virtual function.
        @param functionName The real name of the function.
        @param groupName The name of the group.
        @param functionDict The dictionary used to prevent duplicated function names.
        """
        # Find all the field-value test case functions, expand them and add the
        # single items to the virtual function list.
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the field-value test case set file.
        self._testHelper.pushChangeSet()
        retVal = self._ctx.call(functionName, [])
        self._testHelper.popChangeSet()
 
        filename, testCaseList = retVal if type(retVal) in (list, tuple) else (retVal, None)
 
        groupXMLNode = etree.Element(
            "GroupNode",
            name=virtualFunctionBaseName,
            basename=groupName,
            type="FieldValueTest",
            sourceName=functionName,
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
        # Keep in mind that duplicate names aren't allowed in field-value test case
        # sets either.
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
 
        fvtcs = FieldValueTestCaseSet(self._ctx)
        if not fvtcs.load(filename):
            mevis.MLAB.logError(f"Failed to load the field-value test case set ({filename})!")
            return
 
        if not testCaseList:
            for testcase in sorted(fvtcs.getList()):
                virtualFunctionName = f"{virtualFunctionBaseName}_{testcase}"
                groupXMLNode.append(
                    self.__getFunctionInfo(
                        testcase,
                        virtualFunctionName,
                        (self.callFieldValueTestCasecallFieldValueTestCase, (filename, testcase)),
                        isDisabled=isDisabled,
                        sourceName=functionName,
                    )
                )
        else:
            availableTestCaseList = fvtcs.getList()
            for testcase in sorted(testCaseList):
                if testcase in availableTestCaseList:
                    virtualFunctionName = f"{virtualFunctionBaseName}_{testcase}"
                    groupXMLNode.append(
                        self.__getFunctionInfo(
                            testcase,
                            virtualFunctionName,
                            (self.callFieldValueTestCasecallFieldValueTestCase, (filename, testcase)),
                            isDisabled=isDisabled,
                            sourceName=functionName,
                        )
                    )
                else:
                    mevis.MLAB.logError(f"The field-value test case {testcase} is unknown!")
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __createIterativeTestNode -----------------------------------------------------------{{{-
    
    def __createIterativeTestNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
        # Find all iterative test functions, expand them and add the single items
        # to the virtual function list.
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the iterations and the actual function to call.
        self._testHelper.pushChangeSet()
        retVal = self._ctx.call(functionName, [])
        self._testHelper.popChangeSet()
 
        try:
            iterator, function = retVal
        except:  # pylint: disable=W0702
            mevis.MLAB.logError(
                f"The iterative test function ({functionName}) must return a list or dictionary"
                f" with parameters and a function to call!"
            )
            return
 
        groupXMLNode = etree.Element(
            "GroupNode", name=virtualFunctionBaseName, basename=groupName, type="IterativeTest", sourceName=functionName
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
        # Duplicate item nodes are not possible by design (either an index is used
        # or a dictionary).
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
        regExpFunctionNames = re.compile(r"^[.\w]+$")
 
        # Create virtual functions for given iterative test.
        if isinstance(iterator, (list, tuple)):
            ctr = 0
            # Let the index used to identify the function be zero padded.
            justificationWidth = len(str(len(iterator) - 1))
            for item in iterator:
                # The zero padded identifier.
                virtualFunctionId = str(ctr).rjust(justificationWidth, "0")
                args = [item] if (type(item) not in (list, tuple)) else item
                virtualFunctionName = f"{virtualFunctionBaseName}_{virtualFunctionId}"
                if regExpFunctionNames.match(virtualFunctionName):
                    groupXMLNode.append(
                        self.__getFunctionInfo(
                            virtualFunctionId,
                            virtualFunctionName,
                            (function, args),
                            isDisabled=isDisabled,
                            sourceName=functionName,
                        )
                    )
                else:
                    mevis.MLAB.logError(
                        f"The iterative test function ({functionName}) defines non-alphanumeric virtual "
                        f"function name {virtualFunctionName}!"
                    )
                ctr += 1
        elif isinstance(iterator, dict):
            for item in sorted(iterator.keys()):
                args = [iterator[item]] if (type(iterator[item]) not in (list, tuple)) else iterator[item]
                virtualFunctionName = f"{virtualFunctionBaseName}_{item}"
                if regExpFunctionNames.match(virtualFunctionName):
                    groupXMLNode.append(
                        self.__getFunctionInfo(
                            str(item),
                            virtualFunctionName,
                            (function, args),
                            isDisabled=isDisabled,
                            sourceName=functionName,
                        )
                    )
                else:
                    mevis.MLAB.logError(
                        f"The iterative test function ({functionName}) defines non-alphanumeric virtual "
                        f"function name {virtualFunctionName}!"
                    )
        else:
            mevis.MLAB.logError(
                f"The iterative test function ({functionName}) must return a list or dictionary with parameters!"
            )
 
    # --------------------------------------------------------------------------------------------}}}-
 
    # -- def __createUnitTestWrapperNode -----------------------------------------------------------{{{-
    
    def __createUnitTestWrapperNode(self, virtualFunctionBaseName, functionName, groupName, functionDict, isDisabled):
        """Create the XML information for the given unit test wrapper, which creates a test function for
        every unit test function, similar to an iterative test.
 
        @param virtualFunctionBaseName The base name of the virtual function.
        @param functionName The real name of the function.
        @param groupName The name of the group.
        @param functionDict The dictionary used to prevent duplicated function names.
        """
 
        # Find all unit test wrapper functions, expand them and add the single items
        # to the virtual function list.
        if groupName in functionDict["names"]:
            mevis.MLAB.logError(f"A test with name {groupName} already exists. No duplicate names allowed.")
            return
 
        # Get the unit test and the actual function to call.
        self._testHelper.pushChangeSet()
        suite = self._ctx.call(functionName, [])
        self._testHelper.popChangeSet()
 
        if not isinstance(suite, unittest.TestSuite):
            mevis.MLAB.logError(f"The unit test wrapper function ({functionName}) must return a unittest.TestSuite!")
            return
 
        groupXMLNode = etree.Element(
            "GroupNode",
            name=virtualFunctionBaseName,
            basename=groupName,
            type="UnitTestWrapper",
            sourceName=functionName,
        )
        etree.SubElement(groupXMLNode, "Documentation").text = self.getDocumentation(functionName)
 
        # Duplicate item nodes are not possible by design (either an index is used
        # or a dictionary).
        functionDict["names"].append(groupName)
        functionDict["nodes"][virtualFunctionBaseName] = groupXMLNode
 
        def _isSkipped(test):
            isClassSkipped = getattr(test.__class__, "__unittest_skip__", False)
            testMethod = getattr(test, test._testMethodName)  # pylint: disable=W0212
            isMethodSkipped = getattr(testMethod, "__unittest_skip__", False)
            return isClassSkipped or isMethodSkipped
 
        # Create virtual functions for given unit test wrapper
        def _unpackSuite(suite):
            allTests = []
            for test in suite._tests:  # pylint: disable=W0212
                if isinstance(test, unittest.TestSuite):
                    allTests.extend(_unpackSuite(test))
                else:
                    if not _isSkipped(test):
                        allTests.append(test)
            return allTests
 
        allTests = _unpackSuite(suite)
 
        def _makeName(test):
            testCaseName = test.__class__.__name__
            testFunctionName = test._testMethodName  # pylint: disable=W0212
            return "{0}_{1}".format(testCaseName, testFunctionName)
 
        allTests.sort(key=_makeName)
        testNames = list(map(_makeName, allTests))
 
        # It could be smarter to integrate this more deeply into the callTestFunction part.
        def _runUnitTestFunction(testFunction):
            testResult = UnitTestResult()
            testResult._previousTestClass = self.__previousTestClass
            if self.__dummyTestSuite is None:
                self.__dummyTestSuite = unittest.TestSuite()
            # handle setup of class and module fixtures (copied from unittest.TestSuite):
            self.__dummyTestSuite._tearDownPreviousClass(testFunction, testResult)
            self.__dummyTestSuite._handleModuleFixture(testFunction, testResult)
            self.__dummyTestSuite._handleClassSetUp(testFunction, testResult)
            self.__previousTestClass = testFunction.__class__
            # run test:
            testFunction.run(testResult)
            testResult.logResult("Test was successful")
 
        for testName, testFunction in zip(testNames, allTests):
            testMethod = getattr(testFunction, testFunction._testMethodName)
            funcXMLNode = self.__getFunctionInfo(
                testName,
                "{}_{}".format(virtualFunctionBaseName, testName),
                (_runUnitTestFunction, [testFunction]),
                isDisabled=isDisabled,
            )
            if hasattr(testMethod, "__func__"):  # testMethod is really a method
                funcXMLNode.attrib["sourceFilename"] = testMethod.__func__.__code__.co_filename
                funcXMLNode.attrib["sourceLine"] = str(testMethod.__func__.__code__.co_firstlineno)
            elif hasattr(testMethod, "__code__"):  # testMethod is actually a function
                funcXMLNode.attrib["sourceFilename"] = testMethod.__code__.co_filename
                funcXMLNode.attrib["sourceLine"] = str(testMethod.__code__.co_firstlineno)
            else:  # None or unsupported type - should we raise here?
                funcXMLNode.attrib["sourceFilename"] = ""
                funcXMLNode.attrib["sourceLine"] = "0"
            groupXMLNode.append(funcXMLNode)
 
    def _callableFunctions(self):
        return self._ctx.callableFunctions()
 
    def _buildVirtualFunctions(self, isNewlyCreated):
        """Build an XML node with the virtual functions.
        The result is an XML node that contains information on all virtual functions
        including the information on hierarchy i.e. which nodes are aggregated in a group.
        Returns a Boolean value stating success.
        """
 
        # A list containing the found function names (to prevent duplicates).
        functionInformationDict = {"names": [], "nodes": {}}
 
        regExpDict = {
            "TestFunction": re.compile(r"^(DISABLED_)?TEST(\d*)_(\w+)$"),
            "TestGroup": re.compile(r"^(DISABLED_)?GROUP(\d*)_(\w+)$"),
            "FieldValueTest": re.compile(r"^(DISABLED_)?FIELDVALUETEST(\d*)_(\w+)$"),
            "IterativeTest": re.compile(r"^(DISABLED_)?ITERATIVETEST(\d*)_(\w+)$"),
            "UnitTestWrapper": re.compile(r"^(DISABLED_)?UNITTEST(\d*)_(\w+)$"),
        }
 
        # The result node.
        functionsNode = etree.SubElement(self.__xmlRoot, "TestFunctions")
 
        # Check the optional list of disabled test functions (it is defined by the disabledTestFunction() decorator)
        disabledFunctions = self._ctx.scriptVariable("MLABTC_DISABLED_TEST_FUNCTIONS") or []
 
        # The dictionary of all test functions sorted by type.
        functionDict = {
            "TestFunction": [],
            "TestGroup": [],
            "FieldValueTest": [],
            "IterativeTest": [],
            "UnitTestWrapper": [],
        }
        for functionName in self._callableFunctions():
            for prefix, functionList in (
                ("TEST", functionDict["TestFunction"]),
                ("GROUP", functionDict["TestGroup"]),
                ("FIELDVALUETEST", functionDict["FieldValueTest"]),
                ("ITERATIVETEST", functionDict["IterativeTest"]),
                ("UNITTEST", functionDict["UnitTestWrapper"]),
            ):
                if functionName.startswith(prefix) or functionName.startswith("DISABLED_" + prefix):
                    functionList.append(functionName)
 
        for functionName in sorted(functionDict["TestFunction"]):
            mObj = regExpDict["TestFunction"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of test function invalid ({functionName})")
            else:
                disabledString, orderString, functionBaseName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{functionBaseName}"
                self.__createNormalTestNode(
                    fname, functionName, functionBaseName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["TestGroup"]):
            mObj = regExpDict["TestGroup"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of test group invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                self.__createTestGroupNode(
                    functionName, orderString, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["FieldValueTest"]):
            mObj = regExpDict["FieldValueTest"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of field value test invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{groupName}"
                self.__createFieldValueTestNode(
                    fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["IterativeTest"]):
            mObj = regExpDict["IterativeTest"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of iterative test invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{groupName}"
                self.__createIterativeTestNode(
                    fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        for functionName in sorted(functionDict["UnitTestWrapper"]):
            mObj = regExpDict["UnitTestWrapper"].match(functionName)
            if not mObj:
                mevis.MLAB.logError(f"Name of unit test wrapper invalid ({functionName})")
            else:
                disabledString, orderString, groupName = mObj.groups()
                isDisabled = (disabledString is not None) or (functionName in disabledFunctions)
                fname = f"TEST{orderString}_{groupName}"
                self.__createUnitTestWrapperNode(
                    fname, functionName, groupName, functionInformationDict, isDisabled=isDisabled
                )
 
        # Append nodes in correct order. Essential for other methods to work
        # properly.
        keys = sorted(functionInformationDict["nodes"].keys())
        if len(keys) > 0:
            for functionName in keys:
                functionsNode.append(functionInformationDict["nodes"][functionName])
        elif not isNewlyCreated:
            mevis.MLAB.logError(f"No test functions found in TestCase {self._testCaseName}!")
 
    def __getFunctionInfo(self, functionBaseName, functionName, calledFunction=None, isDisabled=False, sourceName=""):
        """Internal method to add a function to the required data-structures.
 
        @param functionBaseName The virtual function name.
        @param functionName The real name of the function.
        @param calledFunction Information on the function called.
        """
        self.__functionDict.setdefault(functionName, calledFunction)
        functionXMLNode = etree.Element(
            "Function", basename=functionBaseName, name=functionName, sourceName=sourceName, is_disabled=str(isDisabled)
        )
        documentation = self.getDocumentation(functionName)
        etree.SubElement(functionXMLNode, "Documentation").text = documentation if documentation else ""
        return functionXMLNode
 
    def isValid(self):
        """Return True if the loaded test case is syntactically valid and should be executable."""
        return self.__valid
 
    def hasNetwork(self):
        """Return True if the test case has a network attached."""
        return os.path.exists(self.networkFileName())
 
    def networkFileName(self):
        """Returns the absolute file name of the attached network or an empty string if no network is attached."""
        return self._ctx.network().filename()
 
    def openFiles(self):
        """Open the files associated with the test case."""
        typeName = self.getName()
        for fileName in self._ctx.getScriptSourceFiles():
            MLABpriv.openRelatedFile(typeName, fileName)
 
    def openNetwork(self):
        """Open the internal network of the test case."""
        if self._ctx:
            MLABpriv.showIDE()
            self._ctx.showInternalNetwork()
 
    def getType(self):
        """Return the type of this test case (generic or functional)."""
        raise NotImplementedError(f"getType() is not implemented in {self.__class__.__name__}")
 
    def getModule(self):
        """Return the module representing this test case, or None."""
        return self._ctx
 
    def getName(self):
        """Return the name of the test case."""
        return self._testCaseName
 
    def getListOfTestFuncs(self):
        """Get the list of test functions in this test."""
        return sorted(self.__functionDict.keys())
 
    def getPackage(self):
        """Return the identifier of the package the test case is in."""
        return self.__xmlRoot.find("Package").text
 
    def getTestCaseNode(self):
        """Return the XML node with the test case's information."""
        return self.__xmlRoot
 
    def __callSetUp(self):
        """Call the setup function of the test if available."""
        if "setUp" in self._ctx.callableFunctions():
            with self.__testLogger.collect():
                self._ctx.call("setUp", [])
                mevis.MLAB.log(Utils.lineSeparator())
 
    def __callSetUpTestCase(self):
        """Call the setup function of the test case if available."""
        if "setUpTestCase" in self._ctx.callableFunctions() and not self.__setUpTestCaseCalled:
            self.__setUpTestCaseCalled = True
            with self.__setupLogCollector.collect():
                self._ctx.call("setUpTestCase", [])
 
    def __callTearDown(self):
        """Call the teardown function of the test if available."""
        # do not call tearDown after a failed setUp
        if self.__hasSetUpError:
            self.__hasSetUpError = False
            # in the virtual unload testcase the context is None
        elif self._ctx is not None and "tearDown" in self._ctx.callableFunctions():
            with self.__testLogger.collect():
                mevis.MLAB.log(Utils.lineSeparator())
                self._ctx.call("tearDown", [])
 
    def __callTearDownTestCase(self):
        """Call the teardown function of the test case if available."""
        if self._ctx is not None and "tearDownTestCase" in self._ctx.callableFunctions():
            with self.__testLogger.collect():
                self._ctx.call("tearDownTestCase", [])
 
    def __unloadTestCase(self):
        with self.__testLogger.collect():
            if self.__dummyTestSuite:
                # unittest support: clean up after running unittest.TestCases
                testResult = UnitTestResult()
                testResult._previousTestClass = self.__previousTestClass
                self.__dummyTestSuite._tearDownPreviousClass(None, testResult)
                self.__dummyTestSuite._handleModuleTearDown(testResult)
                testResult.logResult("")
                self.__previousTestClass = None
                self.__dummyTestSuite = None
            if TestSupport.Logging.gStopped:
                raise CancelTestException
            else:
                mevis.MLAB.log("Destroying test case context...")
                self.destroyContext()
 
    def run(self, resultNode: etree.Element):
        """Run the test case and store results in the given xml data structure.
 
        @param resultNode The node taking the results (call by reference).
        """
 
        self.activate()
 
        TestSupport.Logging.setStopped(False)
 
        logHandler = self._testHelper.getLogHandler()
 
        availTestFunctions = self.getListOfTestFuncs()
 
        # Build dict mapping test function names to xml nodes.
        funcList = resultNode.findall(".//Function")
        length = len(funcList)
 
        self.__currentResultNode = resultNode
 
        # Iterate over all test functions (sorted by name) that should be tested.
        # The iteration with an index is required to call setUpTestCase and
        # tearDownTestCase properly.
        for i, funcNode in enumerate(funcList):
            funcName = funcNode.get("name")
 
            isLastTestFunction = i == length - 1
            data = RunTestFunctionData(funcNode, funcName, logHandler, isLastTestFunction)
 
            if not self.__prepareRunTestFunction(data, availTestFunctions):
                continue
 
            self.__runTestFunction(data)
            self.__finishRunTestFunction(data)
            if data.cancelled:
                # Do NOT call __tearDownTestCase() when 'Stop on error' flag is set and an error occurred.
                if not (TestSupport.Logging.gStopOnFirstError and logHandler.numErrors() > 0):
                    self.__callTearDownTestCase()
                    self.__unloadTestCase()
                break
 
        self.deactivate()
 
    def __prepareRunTestFunction(self, data, availTestFunctions):
        if self.__perFunctionResultDirectory:
            funcNameForResultDir = data.funcName
        else:
            funcNameForResultDir = None
        self._testHelper.setTestCaseResultDirectory(self.getName().replace("::", os.sep), funcNameForResultDir)
 
        shouldRunTestFunction = True
 
        # Verify that a function with the given name exists.
        if data.funcName not in availTestFunctions and not Utils.isInternalTestCaseFunction(data.funcName):
            mevis.MLAB.logError(f"Function {data.funcName} does not exist in test case {self.getName()}!")
            status = Utils.TEST_STATUS_FUNCTION_NOT_FOUND
            data.funcNode.set("status", str(status))
            shouldRunTestFunction = False
        return shouldRunTestFunction
 
    def __runTestFunction(self, data):
        data.started = time.time()
 
        try:
            data.logHandler.startTestFunction(self.getType(), self.getName(), data.funcName)
            if self.__testCaseListener:
                self.__testCaseListener.startTestCaseFunction(self.getType(), self.getName(), data.funcName)
 
            MLABpriv.clearLogState()
 
            self.__setupTestCaseAndRunTestFunction(data)
 
            if data.funcName == Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION:
                self.__loadLogCollector.logMessages(data.logHandler)
            elif data.funcName == Utils.SETUP_TEST_CASE_FUNCTION:
                self.__setupLogCollector.logMessages(data.logHandler)
            if data.funcName == Utils.TEARDOWN_TEST_CASE_FUNCTION:
                self.__callTearDownTestCase()
            elif data.funcName == Utils.VIRTUAL_UNLOAD_TEST_CASE_TEST_FUNCTION:
                self.__unloadTestCase()
            if TestSupport.Logging.gStopped:
                raise CancelTestException
        except CancelTestException:
            data.cancelled = True
 
    def __finishRunTestFunction(self, data):
        for extraTestCaseResult in self._testHelper.takeExtraTestCaseResults():
            self.addExtraTestCaseResult(extraTestCaseResult)
 
        if not Utils.isInternalTestCaseFunction(data.funcName):
            self.__callTearDown()
 
        match data.funcName:
            case Utils.VIRTUAL_LOAD_TEST_CASE_TEST_FUNCTION:
                currentLogger = self.__loadLogCollector
            case Utils.SETUP_TEST_CASE_FUNCTION:
                currentLogger = self.__setupLogCollector
            case _:
                currentLogger = self.__testLogger
 
        # process events so that any delayed messages are also part of the test function
        mevis.MLAB.processEvents()
        # force messages from other threads to be printed inside of this test function execution
        MLABpriv.flushPendingMessages()
 
        duration = time.time() - data.started
 
        # Gather status and additional information.
        if currentLogger.hasErrors:
            status = Utils.TEST_STATUS_ERROR
        elif currentLogger.hasWarnings:
            status = Utils.TEST_STATUS_WARNING
        else:
            status = Utils.TEST_STATUS_OK
 
        self.__testLogger.clearState()
        data.logHandler.stopTestFunction(status, duration)
        if self.__testCaseListener:
            self.__testCaseListener.endTestCaseFunction(status, duration)
 
        if data.cancelled:
            status = Utils.TEST_STATUS_CANCELLED
        data.funcNode.set("status", str(status))
        data.funcNode.set("duration", str(duration))
        data.funcNode.append(data.logHandler.getMessages())
 
    def __setupTestCaseAndRunTestFunction(self, data):
        if data.funcNode.get("is_disabled", str(False)) == str(False):
            self.__callSetUpTestCase()
            if data.funcName == Utils.SETUP_TEST_CASE_FUNCTION and TestSupport.Logging.gStopped:
                raise CancelTestException
            if not Utils.isInternalTestCaseFunction(data.funcName):
                self.__callSetUp()
                self.__hasSetUpError = MLABpriv.hadLogErrors()
                with self.__testLogger.collect():
                    if self.__hasSetUpError:
                        mevis.MLAB.log("Skipping test after error in setUp()")
                    else:
                        mevis.MLAB.log(f"Calling test function {data.funcName}()")
                        self.__callTestFunction(data.funcName)
        else:
            mevis.MLAB.log(f"Skipping disabled test function {data.funcName}()")
 
    def getDocumentation(self, functionName):
        """Get the documentation for the given test function.
        The found string will be modified such that all trailing whitespace will be removed.
 
        @param functionName The function to gather documentation for.
        @return The function's documentation.
        """
        docu = self._ctx.scriptVariable(f"{functionName}.__doc__")
        testDocu = docu if docu else ""
        return re.sub(r"(?m)^\s+", "", testDocu)