109    def __init__(self, cfgFile, context, offline=False):
 
  121        resultDir = self.
_config.getResultDir()
 
  122        if not os.path.isdir(resultDir):
 
  123            os.makedirs(resultDir)
 
  131        if not mevis.MLABTestCaseDatabase.areTestCasesLoaded():
 
  132            mevis.MLABTestCaseDatabase.loadTestCases()
 
  135        TestHelper.getInstance(context=self.
__ctx, resultDir=self.
_config.getResultDir())
 
 
  152        testType = testCaseNode.get(
"type")
 
  153        moduleName = testCaseNode.get(
"module") 
if testType == 
"GenericTestCase" else None 
  154        return self.
__loadTestCase(testCaseNode.get(
"name"), testType, moduleName)
 
 
  165    def __loadTestCase(self, testCaseName, testCaseType, moduleName=None):
 
  167        if testCaseType == 
"GenericTestCase":
 
  169        elif testCaseType == 
"FunctionalTestCase":
 
  171        elif testCaseType == 
"CodeTestTestCase":
 
  172            testCase = CodeTestTestCase(testCaseName)
 
  174            raise Exception(
"Unsupported test case type %s" % testCaseType)
 
  210    def __doTest(self, testCaseNode, resultNode):
 
  213            maxErrorMessagesPerTestFunction = int(testCaseNode.get(
"maxErrorMessagesPerTestFunction"))
 
  215            maxErrorMessagesPerTestFunction = 10000
 
  217            maxInfoMessagesPerTestFunction = int(testCaseNode.get(
"maxInfoMessagesPerTestFunction"))
 
  219            maxInfoMessagesPerTestFunction = 10000
 
  228            pythonCoverage = CodeCoverage.PythonCoverage(config=self.
_config, testCaseNode=testCaseNode)
 
  229        except CodeCoverage.CoverageException 
as e:
 
  230            pythonCoverage = 
None 
  231            mevis.MLAB.logError(e)
 
  234            bullseyeCoverage = CodeCoverage.BullseyeCoverage(config=self.
_config, testCaseNode=testCaseNode)
 
  235        except CodeCoverage.CoverageException 
as e:
 
  236            bullseyeCoverage = 
None 
  237            mevis.MLAB.logError(e)
 
  244            if not testCaseNode.get(
"status") == Utils.TEST_STATUS_DO_NOT_TEST_MODULE:
 
  245                if pythonCoverage 
and pythonCoverage.isRunning():
 
  248                    resultNode.set(
"pythonCoverage", pythonCoverage.getCoverageBinaryFilePath())
 
  250                if bullseyeCoverage 
and bullseyeCoverage.isRunning():
 
  253                    resultNode.set(
"bullseyeCoverage", bullseyeCoverage.getCoverageBinaryFilePath())
 
  256                logHandler.setErrorLimit(maxErrorMessagesPerTestFunction)
 
  257                logHandler.setInfoLimit(maxInfoMessagesPerTestFunction)
 
  258                TestHelper.getInstance().setLogHandler(logHandler)
 
  260                mevis.MLAB.priv().connect(
"loggingMessage(const QString&)", logHandler.appendMessage)
 
  261                mevis.MLAB.priv().aboutToLogMessage.connect(logHandler.aboutToLogMessage)
 
  262                mevis.MLAB.priv().startLogCollect()
 
  266                mevis.MLAB.priv().stopLogCollect()
 
  267                mevis.MLAB.priv().disconnect(
"loggingMessage(const QString&)", logHandler.appendMessage)
 
  268                mevis.MLAB.priv().aboutToLogMessage.disconnect(logHandler.aboutToLogMessage)
 
  270                TestHelper.getInstance().unsetLogHandler()
 
  273            resultNode.set(
"errorCode", 
"Ok")
 
  275            resultNode.append(testCaseNode)
 
  277            resultNode.set(
"errorCode", 
"TestCaseError")
 
  278            etree.SubElement(resultNode, 
"Error").text = 
"Unknown test case (%s)." % (testCaseNode.get(
"name"))
 
  281            pythonCoverage.stop()
 
  285                bullseyeCoverage.stop()
 
  286            except CodeCoverage.CoverageException 
as e:
 
  287                mevis.MLAB.logError(e)
 
  297    def __buildTestAgenda(self, testCfg, resultNode):
 
  299        def ___addPackageInfoNode(rootNode):
 
  300            packageInfoNode = etree.SubElement(rootNode, 
"PackageSortingInReport")
 
  302                "PackagesByNumberOfDependencies", 
",".join(PackageSorter.byNumberOfDependencies.packages)
 
  305                "PackageGroupsByNumberOfDependencies", 
",".join(PackageSorter.byNumberOfDependencies.packageGroups)
 
  308        testRunNode = etree.Element(
"TestRun")
 
  312            "Packages": {
"Tested": [], 
"Available": []},
 
  322        for testGroupNode 
in testCfg.findall(
"TestGroups/TestGroup"):
 
  323            self.
__testCfg[
"TestGroups"].append(testGroupNode.get(
"name"))
 
  326        for testSuiteNode 
in testCfg.findall(
"TestSuites/TestSuite"):
 
  327            testSuite = testSuiteNode.get(
"name")
 
  328            testCaseNames = mevis.MLABTestCaseDatabase.allTestCaseSuites().get(testSuite, {}).get(
"testCases", [])
 
  329            testCasesNode = testCfg.find(
"TestCases")
 
  330            for name 
in testCaseNames:
 
  331                etree.SubElement(testCasesNode, 
"TestCase", {
"name": name})
 
  334        testRunNode.set(
"platform", mevis.MLAB.systemId())
 
  335        testRunNode.set(
"compiler", mevis.MLAB.compilerInfo())
 
  337        testRunNode.set(
"builddate", mevis.MLAB.dateOfBuildJob().date().toString(
"yyyy-MM-dd"))
 
  338        testRunNode.set(
"version", mevis.MLAB.priv().variable(
"Version", 
False))
 
  341        infoNode = etree.SubElement(testRunNode, 
"Information")
 
  342        testResultNode = etree.SubElement(testRunNode, 
"Results")
 
  343        etree.SubElement(infoNode, 
"TestCases")
 
  344        etree.SubElement(infoNode, 
"Modules")
 
  345        ___addPackageInfoNode(infoNode)
 
  353        for testCaseNode 
in infoNode.findall(
"./TestCases/TestCase"):
 
  354            if testCaseNode.get(
"type") == 
"GenericTestCase":
 
  360        resultNode.set(
"errorCode", 
"Ok")
 
  361        resultNode.append(testRunNode)
 
  362        resultNode.append(testCfg)
 
  369    def __buildTestCaseAgenda(self, requestNode, resultNode):
 
  371            parameterNode = requestNode.find(
"Parameters")
 
  372            packagesToTest = parameterNode.get(
"packages", 
"").split(
",")
 
  373            modulesToTest = parameterNode.get(
"modules", 
"").split(
",")
 
  374            testCaseName = parameterNode.find(
"TestCase").get(
"name")
 
  375            testResultsNode = etree.SubElement(resultNode, 
"Results")
 
  377                testCaseName, testResultsNode, packagesToTest, modulesToTest
 
  379            testCase = testResultsNode.find(
"TestCase")
 
  381            if testCase 
is not None and testCase.get(
"status", 
"") == str(Utils.TEST_STATUS_DO_NOT_TEST_MODULE):
 
  382                functionListNode = testCaseAgenda.find(
"TestFunctions")
 
  383                functionNodes = functionListNode.findall(
"Function")
 
  384                for functionNode 
in functionNodes:
 
  385                    functionListNode.remove(functionNode)
 
  386            assert testCaseAgenda
 
  387            resultNode.append(testCaseAgenda)
 
  389                etree.SubElement(resultNode, 
"Error").text = 
"Failed to load the test case" 
  390            resultNode.set(
"errorCode", status)
 
  392            resultNode.set(
"errorCode", 
"Failed")
 
  393            etree.SubElement(resultNode, 
"Error").text = traceback.format_exc()
 
  402    def __verifyTestCaseList(self, testCfg, testRunNode):
 
  404        availTestCaseList = []
 
  405        for package 
in self.
__testCfg[
"Packages"][
"Tested"]:
 
  406            availTestCaseList.extend(mevis.MLABTestCaseDatabase.allTestCasesForPackage(package))
 
  408                availTestCaseList.extend(getCodeTestsByPackage().get(package, {}).keys())
 
  411        testCaseListNode = testCfg.find(
"TestCases")
 
  412        if len(testCaseListNode.findall(
"TestCase")) == 0:
 
  413            for testCaseName 
in availTestCaseList:
 
  414                etree.SubElement(testCaseListNode, 
"TestCase", name=testCaseName)
 
  417        testInfoNode = testRunNode.find(
"Information/TestCases")
 
  419        for testCaseNode 
in testCfg.findall(
"TestCases/TestCase"):
 
  420            testCaseName = testCaseNode.get(
"name")
 
  421            if testCaseName 
in availTestCaseList:
 
  426                        "TestCase %s not in tested groups (%s)." 
  427                        % (testCaseName, 
", ".join(self.
__testCfg[
"TestGroups"]))
 
  429                    testCaseNode.set(
"isNotInTestGroups", str(
True))
 
  431                    testCaseNode.set(
"type", infoDict[
"type"])
 
  432                    testInfoNode.append(testCaseNode)
 
  433                    testCaseNode.set(
"status", str(Utils.TEST_STATUS_OK))
 
  435                mevis.MLAB.logError(
"test case not found %s %s" % (testCaseName, availTestCaseList))
 
  436                testCaseNode.set(
"status", str(Utils.TEST_STATUS_ERROR))
 
  442    def __loadAndVerifyTestCase(self, testCaseName, testResultNode, packagesToTest, modulesToTest):
 
  443        resolvedPackagesToTest = []
 
  444        for package 
in packagesToTest:
 
  445            pGroup, pName = package.split(
"/")
 
  446            if pGroup == 
"*" and pName == 
"*":
 
  447                for package 
in mevis.MLABPackageManager.packages():
 
  448                    resolvedPackagesToTest.append(package.packageIdentifier())
 
  450                for package 
in mevis.MLABPackageManager.packages():
 
  451                    if package.packageGroup() == pGroup:
 
  452                        resolvedPackagesToTest.append(package.packageIdentifier())
 
  454                resolvedPackagesToTest.append(package)
 
  456            testCaseName, testResultNode, resolvedPackagesToTest, modulesToTest
 
  458        return testCaseNode, status
 
  462    def __getTestCaseInfo(self, testCaseName):
 
  463        infoDict = mevis.MLABTestCaseDatabase.testCaseInfo(testCaseName)
 
  465            codeTest = getCodeTest(testCaseName)
 
  467                infoDict = codeTest.getInfoDict()
 
  475    def __verifyTestCase(self, testCaseName, testResultNode, packagesToTest, modulesToTest):
 
  479        testType = infoDict[
"type"]
 
  484            mevis.MLAB.log(
"Trying to load test case %s" % testCaseName)
 
  487            mevis.MLAB.log(
"Loading complete")
 
  490            return None, 
"Failed" 
  493        if testType == 
"GenericTestCase":
 
  494            modulesToTest, modulesNotToTest = testCase.getModulesToTest(modulesToTest, packagesToTest)
 
  495            for moduleName 
in modulesToTest:
 
  496                etree.SubElement(testResultNode, 
"TestCase", name=testCaseName, type=testType, module=moduleName)
 
  497            for moduleName 
in modulesNotToTest:
 
  504                    status=str(Utils.TEST_STATUS_DO_NOT_TEST_MODULE),
 
  507            etree.SubElement(testResultNode, 
"TestCase", name=testCaseName, type=testType)
 
  509        return testCase.getTestCaseNode(), 
"Ok" 
  521    def __verifyTestGroups(self, infoDict, testGroupList):
 
  522        testCaseGroupList = infoDict[
"testGroups"].lower() 
if "testGroups" in infoDict 
else "" 
  523        testCaseGroupList = testCaseGroupList.split()
 
  525        if "disabled" in testCaseGroupList 
or (
"needspublicsdk" in testCaseGroupList 
and not _isPublicSDK):
 
  527        testGroupList = [tg.lower() 
for tg 
in testGroupList]
 
  529            if "automatic" in testGroupList:
 
  530                if "manual" in testCaseGroupList:
 
  533            for testGroup 
in testGroupList:
 
  534                if testGroup 
in testCaseGroupList:
 
  548    def __verifyPackages(self, testCfg):
 
  549        packageRoot = testCfg.find(
"Packages")
 
  553        tPkgNode = packageRoot.find(
"Tested")
 
  554        aPkgNode = packageRoot.find(
"Available")
 
  555        iPkgNode = packageRoot.find(
"Ignored")
 
  558        if len(iPkgNode) > 0:
 
  560        ignoredPackageIDs = set()
 
  561        for ignoredPackage 
in iPkgNode:
 
  562            ignoredPackageIDs.add(ignoredPackage.get(
"name"))
 
  568        for packageNode 
in tPkgNode.findall(
"Package"):
 
  569            if packageNode.get(
"status") 
in (
"Ok", 
"Added"):
 
  570                tPkgSet.add(packageNode.get(
"name"))
 
  573        for packageNode 
in aPkgNode.findall(
"Package"):
 
  574            if packageNode.get(
"status") 
in (
"Ok", 
"Added"):
 
  575                aPkgSet.add(packageNode.get(
"name"))
 
  578        self.
__testCfg[
"Packages"][
"Tested"] = list(tPkgSet)
 
  579        self.
__testCfg[
"Packages"][
"Available"] = list(tPkgSet.union(aPkgSet))
 
  582        for item 
in tPkgSet.difference(aPkgSet):
 
  583            etree.SubElement(aPkgNode, 
"Package", name=item, status=
"Ok")
 
  591    def __verifyPackageListNode(self, packageListNode, ignoredPackageIDs=[]):
 
  593        if len(packageListNode) == 0:
 
  594            etree.SubElement(packageListNode, 
"Package", name=
"*/*")
 
  596        for packageNode 
in packageListNode:
 
  597            packageID = packageNode.get(
"name")
 
  599                pGroup, pName = packageID.split(
"/")
 
  601                packageNode.set(
"status", 
"Invalid")
 
  607                    for package 
in mevis.MLABPackageManager.packages():
 
  608                        packageID = package.packageIdentifier()
 
  609                        if not packageID 
in ignoredPackageIDs:
 
  610                            etree.SubElement(packageListNode, 
"Package", name=packageID, status=
"Added")
 
  612                    packageNode.set(
"status", 
"Substituted")
 
  614                    packageNode.set(
"status", 
"Failed")
 
  616                packageNode.set(
"status", 
"Substituted")
 
  617                for package 
in mevis.MLABPackageManager.packagesByGroup(pGroup):
 
  618                    packageID = package.packageIdentifier()
 
  619                    if not packageID 
in ignoredPackageIDs:
 
  620                        etree.SubElement(packageListNode, 
"Package", name=packageID, status=
"Added")
 
  624                if mevis.MLABPackageManager.packageByIdentifier(packageID):
 
  625                    packageNode.set(
"status", 
"Ok")
 
  627                    packageNode.set(
"status", 
"Failed")
 
  637    def __collectModulesToTest(self, testCfg):
 
  638        modulesNode = testCfg.find(
"Modules")
 
  642        for filterNode 
in modulesNode.findall(
"Filter"):
 
  643            key, valueList = filterNode.text.split(
":")
 
  645            for value 
in valueList.split(
","):
 
  646                filterDict[key].append(value)
 
  649        availableModuleList = []
 
  650        for packageName 
in self.
__testCfg[
"Packages"][
"Tested"]:
 
  651            availableModuleList.extend(mevis.MLAB.allModulesForPackageIdentifier(packageName))
 
  654        if len(modulesNode.findall(
"Module")) == 0:
 
  655            for moduleName 
in availableModuleList:
 
  656                etree.SubElement(modulesNode, 
"Module", name=moduleName)
 
  659        for moduleNode 
in modulesNode.findall(
"Module"):
 
  660            moduleName = moduleNode.get(
"name")
 
  661            moduleInfo = mevis.MLAB.moduleInfo(moduleName)
 
  663            if moduleInfo 
and moduleName 
in availableModuleList:
 
  667                    if not f 
in moduleInfo 
or not [x 
for x 
in filterDict[f] 
if re.search(x, moduleInfo[f])]:
 
  671                    self.
__testCfg[
"Modules"].append(moduleName)
 
  672                    moduleNode.set(
"status", str(Utils.TEST_STATUS_OK))
 
  674                    moduleNode.set(
"isModuleFiltered", str(
True))
 
  676                modulesNode.set(
"status", str(Utils.TEST_STATUS_ERROR))
 
  682    def __buildModuleInfo(self, modulesInfoNode):
 
  683        for moduleName 
in self.
__testCfg[
"Modules"]:
 
  684            moduleInfo = mevis.MLAB.moduleInfo(moduleName)
 
  685            newModuleNode = etree.SubElement(modulesInfoNode, 
"Module", name=moduleName)
 
  686            for key 
in moduleInfo:
 
  687                value = moduleInfo[key]
 
  688                etree.SubElement(newModuleNode, key, type=value.__class__.__name__).text = 
unicode(value)
 
  696    def __setPackageList(self, xmlNode, resultNode):
 
  698        for item 
in xmlNode.findall(
"Package"):
 
  699            packageList.append(item.get(
"name"))
 
  700        TestHelper.getInstance().setPackageList(packageList)
 
  701        resultNode.set(
"errorCode", 
"Ok")
 
  711        resultNode = etree.Element(
"Result")
 
  712        if not TestHelper.getInstance().getGlobalContext():
 
  714            mevis.MLAB.logWarning(
"Test case manager has been closed - aborting test execution")
 
  716            command = requestNode.get(
"type")
 
  717            mevis.MLAB.log(
"Running command '%s'" % command)
 
  720            if command == 
"Quit":
 
  721                resultNode.set(
"errorCode", 
"Ok")
 
  723            elif command == 
"DoTest":
 
  724                testCaseNode = requestNode.find(
"TestCase")
 
  725                if self.
__com and testCaseNode.get(
"requestProgress") == 
"1":
 
  728                self.
__doTest(testCaseNode, resultNode)
 
  730                if self.
__com and testCaseNode.get(
"requestProgress") == 
"1":
 
  733            elif command == 
"SetPackageList":
 
  735            elif command == 
"BuildTestAgenda":
 
  737            elif command == 
"BuildTestCaseAgenda":
 
  740                raise Exception(
"Unknown command: %s" % (command))
 
  742            resultNode.set(
"errorCode", 
"Exception")
 
  743            etree.SubElement(resultNode, 
"Error").text = traceback.format_exc()
 
  744            mevis.MLAB.log(traceback.format_exc())
 
  747        content = etree.tostring(resultNode).decode()
 
  748        mevis.MLAB.log(
"Result content: %s%s" % (content[:150], 
" [...]" if len(content) > 150 
else ""))
 
 
  758        TestHelper.getInstance().runTestInSecureMode()
 
  760            request = self.
__com.recv(20)
 
  761            if request 
in (
"", 
None):
 
  762                mevis.MLAB.logError(self.
__com.getLastErrorWithMessage()[1])
 
  765            requestNode = etree.fromstring(request)
 
  768            if not self.
__com.send(etree.tostring(resultNode), 5):
 
  769                mevis.MLAB.logError(
"Couldn't send result to request %s" % (request))
 
  771        TestHelper.getInstance().runTestInCurrentMlabInstance()
 
  772        self.
__com.disconnect()