PythonUnitTest.py

Go to the documentation of this file.

#
# Copyright 2014, MeVis Medical Solutions AG
#
# The user may use this file in accordance with the license agreement provided with
# the Software or, alternatively, in accordance with the terms contained in a
# written agreement between the user and MeVis Medical Solutions AG.
#
# For further information use the contact form at https://www.mevislab.de/contact
#
 
"""Tools for running MeVisLab tests as Python unit tests
 
The functions, decorators and classes defined in this module allow you to
wrap your MeVisLab FunctionalTestCase in a unittest.TestCase so that the
test case can be executed using the standard Python unit testing framework.
 
To wrap a FunctionalTestCase in a unittest.TestCase add the following at the
end of the test case file.
 
from TestSupport import PythonUnitTest
 
@PythonUnitTest.addTestsToMeVisLabTestCase(globals())
class NameOfFunctionalTestCase(PythonUnitTest.MeVisLabTestCase):
    pass
 
@note: NameOfFunctionalTestCase has to be identical with the name of the
FunctionalTestCase specified in the .def file. Otherwise, the MeVisLab
TestCenter that is used to execute the tests will not find the tests.
 
The above is sufficient to allow PyCharm(TM) to run your MeVisLab functional
tests as any other Python unit tests.
 
If you also want to run your tests from the command line, then, additionally,
add the following at the end of the file with the MeVisLab tests (just as for
any old Python unit test).
 
if __name__ == '__main__':
    import unittest
    unittest.main()
 
@note: Wrapping ITERATIVETEST functions only works if the generation of the
test dictionary returned by the ITERATIVETEST function does not require
functionality that only works if the test runs in MeVisLab. For example,
TestSupport.Base.getDataDirectory() cannot be used in ITERATIVETEST. Instead
you can use something like
 
    if __file__:
        # use __file__ if running outside of MeVisLab (where Base.getDataDirectory() doesn't work)
        dataDirectory = os.path.join(os.path.dirname(__file__), 'Data')
    else:
        # use Base.getDataDirectory() if running in MeVisLab (where __file__ is not set)
        dataDirectory = Base.getDataDirectory()
 
"""
 
import logging
import os
import re
import shutil
import sys
import tempfile
import unittest
from xml.etree import cElementTree as ElementTree
 
 
mlabroot = os.environ.get('MLAB_ROOT')
 
 
@unittest.skip('Test is marked as DISABLED')
def skippedDummyTest(_):
    pass
 
 
NormalTestRegExp = re.compile(r'^(?:DISABLED_)?TEST(\d*_\w+)$')
IterativeTestRegExp = re.compile(r'^(?:DISABLED_)?ITERATIVETEST(\d*_\w+)$')
 
 
def addTestsToMeVisLabTestCase(globals):
    def inner(testCaseClass):
        # do nothing if we are running inside MeVisLab
        runningInsideMeVisLab = hasattr(globals.get('ctx'), 'name')
        if runningInsideMeVisLab:
            return testCaseClass
 
        MLABTC_DISABLED_TEST_FUNCTIONS = globals.get('MLABTC_DISABLED_TEST_FUNCTIONS', [])
        for name in globals:
            skip = name.startswith('DISABLED_') or name in MLABTC_DISABLED_TEST_FUNCTIONS
            m = NormalTestRegExp.match(name)
            if m:
                testName = m.group(1)
                testCaseClass.addTest(testName, skip)
                continue
            m = IterativeTestRegExp.match(name)
            if m:
                basename = m.group(1)
                testDict = globals[name]()[0]
                if isinstance(testDict, (list, tuple)):
                    # test ids are zero-padded numbers
                    width = len(str(len(testDict)-1))
                    testDict = [str(n).zfill(width) for n in range(len(testDict))]
                for key in testDict:
                    testName = '%s_%s' % (basename, key)
                    testCaseClass.addTest(testName, skip)
                continue
        return testCaseClass
    return inner
 
 
class MeVisLabTestCase(unittest.TestCase):
    master = None
    tempDirectory = None
 
    def __init__(self, methodName='runTest'):
        super(MeVisLabTestCase, self).__init__(methodName)
 
        if mlabroot is None:
            raise Exception('MLAB_ROOT is not defined')
 
        testCenterAdvancedPath = os.path.join(mlabroot, 'MeVisLab', 'Standard', 'Modules', 'Macros', 'Tests', 'TestCenterAdvanced')
        if not testCenterAdvancedPath in sys.path:
            sys.path.insert(0, testCenterAdvancedPath)
 
    @classmethod
    def setUpClass(cls):
        cls.tempDirectory = tempfile.mkdtemp()
        # print 'Using temp folder %s' % cls.tempDirectory
        configFilename = os.path.join(cls.tempDirectory, 'tc_config.xml')
        writeTestCenterConfigFile(configFilename,
                                  executablePath=os.path.join(mlabroot, 'MeVisLab'),
                                  resultDirectory=cls.tempDirectory,
                                  reportDirectory=cls.tempDirectory)
        # with open(configFilename, 'r') as f:
        #     print f.read()
 
        env = _getEnvironmentForTestCenter(mlabroot)
 
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s %(levelname)-8s %(message)s')
        logging.getLogger('TestCenter').disabled = True
 
        from Master import Master
        cls.master = Master(cfgFile=configFilename, slaveEnvironment=env)
 
    @classmethod
    def tearDownClass(cls):
        if cls.master is not None:
            cls.master.stopSlave()
        if cls.tempDirectory is not None:
            shutil.rmtree(cls.tempDirectory)
        pass
 
    @classmethod
    def addTest(cls, testName, skip):
        testFuncName = 'test%s' % testName
        if skip:
            testFunc = skippedDummyTest
        else:
            testIdentifier = (cls.__name__, 'TEST%s' % testName)
            testFunc = cls.createRunTestMethod(testFuncName, testIdentifier)
        setattr(cls, testFuncName, testFunc)
 
    @classmethod
    def createRunTestMethod(cls, testFuncName, testIdentifier):
        def formatStack(event):
            stack = event.find('Stack')
            stackText = stack.text if stack is not None else 'No stack trace available'
            return 'Stack trace (most recent call last):\n%s' % stackText
 
        def runTestMethod(testCaseInstance):
            cls.master.setTestCases([cls.__name__])
            testFunctionDict = {testIdentifier[0]: [testIdentifier[1]]}
            result = testCaseInstance.master.run(testFunctionDict, stopSlaveAfterTestRun=False)
            testCaseInstance.master.reset()
 
            if result is None:
              raise Exception("Test failed: TestRun did not return anything.")
 
            # find the results of running the test function in the TestRun XML tree
            functionNode = result.find('Results/TestCase/Function')
            if functionNode is None:
                raise Exception('No function found in\n%s' % ElementTree.tostring(result))
            # print ElementTree.tostring(functionNode)
            if functionNode.get('is_disabled') == 'True':
                raise unittest.SkipTest('Skipped for unknown reasons. No test method should have been created for this test.')
 
            messages = []
            status = int(functionNode.get('status'))
            if status == 2:
                messages.append('%s failed' % testIdentifier[1])
            if status == 3:
                messages.append('Test timed out')
            if status == 4:
                messages.append('MeVisLab crashed')
 
            # print all non-Info events
            for event in functionNode.iter('Event'):
                eventType = event.get('type')
                # filename = event.get('file')
                # lineNumber = event.get('line')
                timestamp = event.get('timestamp')
                eventMessageText = _stripTags(event.find('Message').text)
                if eventType == 'Info':
                    pass
                elif eventType == 'Warning':
                    print('%s %s: %s' % (timestamp, eventType, eventMessageText))
                elif eventType in ['Error', 'ML Error', 'Timeout', 'Crashed', 'ScriptError', 'SemanticError']:
                    messages.append('%s: %s' % (eventType, eventMessageText))
                    messages.append(formatStack(event))
                else:
                    print('Unknown event type "%s"' % eventType)
                    print('%s %s: %s' % (timestamp, eventType, eventMessageText))
 
            # fail test if test failed or had error in MeVisLab
            if status == 2:
                testCaseInstance.fail('\n'.join(messages))
            if status == 3 or status == 4:
                raise Exception('\n'.join(messages))
 
        runTestMethod.__name__ = testFuncName
        return runTestMethod
 
 
def writeTestCenterConfigFile(configFilename,
                              executablePath='',
                              resultDirectory='',
                              reportDirectory=''):
    import Config
    config = Config.Configuration(configFilename, autoSave=False)
    config.setMLABExecutablePath(executablePath)
    config.setResultDir(resultDirectory)
    config.getReportOptions().directory = reportDirectory
    config.save()
 
 
def _getEnvironmentForTestCenter(mlabroot):
    buildSystemRoot = os.path.join(mlabroot, 'MeVis', 'BuildSystem', 'Sources')
    sys.path.insert(0, buildSystemRoot)
 
    import BuildSystem.SystemUtils
 
    # remove path again to prevent name clash of IPC modules
    sys.path.remove(buildSystemRoot)
 
    testCenterEnvironment = dict(os.environ)
    if testCenterEnvironment.get('PYCHARM_HOSTED') == '1':
        # delete the PYTHONPATH set by PyCharm because it is wrong for TestCenter.py
        del testCenterEnvironment['PYTHONPATH']
 
    requiredPackages = ['MeVis/ThirdParty', 'MeVis/Foundation', 'MeVisLab/IDE']
    additionalBinPaths = [os.path.join(mlabroot, package, 'bin') for package in requiredPackages]
    additionalBinPaths.append(os.path.join(mlabroot, 'MeVis/ThirdParty/Sources/Qt4/qt/bin'))
    additionalLibPaths = [os.path.join(mlabroot, package, 'lib') for package in requiredPackages]
    additionalLibPaths.append(os.path.join(mlabroot, 'MeVis/ThirdParty/Python'))
    additionalLibPaths.append(os.path.join(mlabroot, 'MeVis/ThirdParty/Sources/Qt4/qt/lib'))
    additionalBinPaths = [os.path.normpath(path) for path in additionalBinPaths]
    additionalLibPaths = [os.path.normpath(path) for path in additionalLibPaths]
    BuildSystem.SystemUtils.addPathsToExecutableSearchPaths(additionalBinPaths, environmentDict=testCenterEnvironment)
    BuildSystem.SystemUtils.addPathsToLibrarySearchPaths(additionalLibPaths, environmentDict=testCenterEnvironment)
 
    return testCenterEnvironment
 
 
_stripTagsRE = re.compile(r'(<!--.*?-->|<[^>]*>)')
 
def _stripTags(s):
    """ Strips HTML tags from the given string.
        Warning: This function is not fool-proof. It is only meant to handle well-formed HTML in MeVisLab log messages.
    """
    return _stripTagsRE.sub('', s)