103 def __init__ (self, cfgFile, context, offline=False):
115 resultDir = self.
_config.getResultDir()
116 if not os.path.isdir(resultDir):
117 os.makedirs(resultDir)
125 if not mevis.MLABTestCaseDatabase.areTestCasesLoaded():
126 mevis.MLABTestCaseDatabase.loadTestCases()
129 TestHelper.getInstance(context=self.
__ctx, resultDir=self.
_config.getResultDir())
145 testType = testCaseNode.get(
'type')
146 moduleName = testCaseNode.get(
'module')
if testType ==
"GenericTestCase" else None
147 return self.
__loadTestCase(testCaseNode.get(
'name'), testType, moduleName)
157 def __loadTestCase (self, testCaseName, testCaseType, moduleName=None):
159 if testCaseType ==
'GenericTestCase':
161 elif testCaseType ==
'FunctionalTestCase':
163 elif testCaseType ==
'CodeTestTestCase':
164 testCase = CodeTestTestCase(testCaseName)
166 raise Exception(
'Unsupported test case type %s' % testCaseType)
203 maxErrorMessagesPerTestFunction = int(testCaseNode.get(
"maxErrorMessagesPerTestFunction"))
205 maxErrorMessagesPerTestFunction = 10000
207 maxInfoMessagesPerTestFunction = int(testCaseNode.get(
"maxInfoMessagesPerTestFunction"))
209 maxInfoMessagesPerTestFunction = 10000
218 pythonCoverage = CodeCoverage.PythonCoverage(config=self.
_config,
219 testCaseNode=testCaseNode)
220 except CodeCoverage.CoverageException
as e:
221 pythonCoverage =
None
222 mevis.MLAB.logError(e)
225 bullseyeCoverage = CodeCoverage.BullseyeCoverage(config=self.
_config,
226 testCaseNode=testCaseNode)
227 except CodeCoverage.CoverageException
as e:
228 bullseyeCoverage =
None
229 mevis.MLAB.logError(e)
236 if not testCaseNode.get(
"status") == Utils.TEST_STATUS_DO_NOT_TEST_MODULE:
237 if pythonCoverage
and pythonCoverage.isRunning():
240 resultNode.set(
'pythonCoverage', pythonCoverage.getCoverageBinaryFilePath())
242 if bullseyeCoverage
and bullseyeCoverage.isRunning():
245 resultNode.set(
'bullseyeCoverage', bullseyeCoverage.getCoverageBinaryFilePath())
248 logHandler.setErrorLimit(maxErrorMessagesPerTestFunction)
249 logHandler.setInfoLimit(maxInfoMessagesPerTestFunction)
250 TestHelper.getInstance().setLogHandler(logHandler)
252 mevis.MLAB.priv().connect(
"loggingMessage(const QString&)", logHandler.appendMessage)
253 mevis.MLAB.priv().aboutToLogMessage.connect(logHandler.aboutToLogMessage)
254 mevis.MLAB.priv().startLogCollect()
258 mevis.MLAB.priv().stopLogCollect()
259 mevis.MLAB.priv().disconnect(
"loggingMessage(const QString&)", logHandler.appendMessage)
260 mevis.MLAB.priv().aboutToLogMessage.disconnect(logHandler.aboutToLogMessage)
262 TestHelper.getInstance().unsetLogHandler()
265 resultNode.set(
'errorCode',
'Ok')
267 resultNode.append(testCaseNode)
269 resultNode.set(
'errorCode',
'TestCaseError')
270 etree.SubElement(resultNode,
'Error').text =
"Unknown test case (%s)." % (testCaseNode.get(
'name'))
273 pythonCoverage.stop()
277 bullseyeCoverage.stop()
278 except CodeCoverage.CoverageException
as e:
279 mevis.MLAB.logError(e)
289 def __buildTestAgenda (self, testCfg, resultNode):
291 def ___addPackageInfoNode( rootNode ):
292 packageInfoNode = etree.SubElement( rootNode,
'PackageSortingInReport')
293 packageInfoNode.set(
'PackagesByNumberOfDependencies',
",".join( PackageSorter.byNumberOfDependencies.packages ) )
294 packageInfoNode.set(
'PackageGroupsByNumberOfDependencies',
",".join( PackageSorter.byNumberOfDependencies.packageGroups ) )
296 testRunNode = etree.Element(
'TestRun')
299 self.
__testCfg = {
'Packages': {
'Tested': [],
'Available': [] },
308 for testGroupNode
in testCfg.findall(
'TestGroups/TestGroup'):
309 self.
__testCfg[
'TestGroups'].append(testGroupNode.get(
'name'))
312 for testSuiteNode
in testCfg.findall(
'TestSuites/TestSuite'):
313 testSuite = testSuiteNode.get(
'name')
314 testCaseNames = mevis.MLABTestCaseDatabase.allTestCaseSuites().get(testSuite, {}).get(
"testCases", [])
315 testCasesNode = testCfg.find(
'TestCases')
316 for name
in testCaseNames:
317 etree.SubElement(testCasesNode,
'TestCase', {
'name': name})
320 testRunNode.set(
'platform', mevis.MLAB.systemId())
321 testRunNode.set(
'compiler', mevis.MLAB.compilerInfo())
323 testRunNode.set(
'builddate', mevis.MLAB.dateOfBuildJob().date().toString(
"yyyy-MM-dd"))
324 testRunNode.set(
'version', mevis.MLAB.priv().variable(
'Version',
False))
327 infoNode = etree.SubElement(testRunNode,
'Information')
328 testResultNode = etree.SubElement(testRunNode,
'Results')
329 etree.SubElement(infoNode,
'TestCases')
330 etree.SubElement(infoNode,
'Modules')
331 ___addPackageInfoNode( infoNode )
339 for testCaseNode
in infoNode.findall(
"./TestCases/TestCase"):
340 if testCaseNode.get(
"type") ==
"GenericTestCase":
346 resultNode.set(
'errorCode',
'Ok')
347 resultNode.append(testRunNode)
348 resultNode.append(testCfg)
354 def __buildTestCaseAgenda(self, requestNode, resultNode):
356 parameterNode = requestNode.find(
'Parameters')
357 packagesToTest = parameterNode.get(
'packages',
'').split(
',')
358 modulesToTest = parameterNode.get(
'modules',
'').split(
',')
359 testCaseName = parameterNode.find(
'TestCase').get(
'name')
360 testResultsNode = etree.SubElement(resultNode,
'Results')
361 testCaseAgenda, status = self.
__loadAndVerifyTestCase(testCaseName, testResultsNode, packagesToTest, modulesToTest)
362 testCase = testResultsNode.find(
'TestCase')
364 if testCase
is not None and testCase.get(
'status',
"") == str(Utils.TEST_STATUS_DO_NOT_TEST_MODULE):
365 functionListNode = testCaseAgenda.find(
'TestFunctions')
366 functionNodes = functionListNode.findall(
'Function')
367 for functionNode
in functionNodes:
368 functionListNode.remove(functionNode)
369 assert(testCaseAgenda)
370 resultNode.append(testCaseAgenda)
372 etree.SubElement(resultNode,
'Error').text =
'Failed to load the test case'
373 resultNode.set(
'errorCode', status)
375 resultNode.set(
'errorCode',
'Failed')
376 etree.SubElement(resultNode,
'Error').text = traceback.format_exc()
384 def __verifyTestCaseList (self, testCfg, testRunNode):
386 availTestCaseList = []
387 for package
in self.
__testCfg[
'Packages'][
'Tested']:
388 availTestCaseList.extend(mevis.MLABTestCaseDatabase.allTestCasesForPackage(package))
390 availTestCaseList.extend(getCodeTestsByPackage().get(package,{}).keys())
393 testCaseListNode = testCfg.find(
'TestCases')
394 if len(testCaseListNode.findall(
'TestCase')) == 0:
395 for testCaseName
in availTestCaseList:
396 etree.SubElement(testCaseListNode,
'TestCase', name=testCaseName)
399 testInfoNode = testRunNode.find(
'Information/TestCases')
401 for testCaseNode
in testCfg.findall(
'TestCases/TestCase'):
402 testCaseName = testCaseNode.get(
'name')
403 if testCaseName
in availTestCaseList:
407 mevis.MLAB.logError(
"TestCase %s not in tested groups (%s)." % (testCaseName,
", ".join(self.
__testCfg[
'TestGroups'])))
408 testCaseNode.set(
'isNotInTestGroups', str(
True))
410 testCaseNode.set(
'type', infoDict[
'type'])
411 testInfoNode.append(testCaseNode)
412 testCaseNode.set(
'status', str(Utils.TEST_STATUS_OK))
414 mevis.MLAB.logError(
"test case not found %s %s" %( testCaseName, availTestCaseList))
415 testCaseNode.set(
'status', str(Utils.TEST_STATUS_ERROR))
420 def __loadAndVerifyTestCase(self, testCaseName, testResultNode, packagesToTest, modulesToTest):
421 resolvedPackagesToTest = []
422 for package
in packagesToTest:
423 pGroup, pName = package.split(
'/')
424 if pGroup ==
"*" and pName ==
"*":
425 for package
in mevis.MLABPackageManager.packages():
426 resolvedPackagesToTest.append(package.packageIdentifier())
428 for package
in mevis.MLABPackageManager.packages():
429 if package.packageGroup() == pGroup:
430 resolvedPackagesToTest.append(package.packageIdentifier())
432 resolvedPackagesToTest.append(package)
433 testCaseNode, status = self.
__verifyTestCase(testCaseName, testResultNode, resolvedPackagesToTest, modulesToTest)
434 return testCaseNode, status
437 def __getTestCaseInfo(self, testCaseName):
438 infoDict = mevis.MLABTestCaseDatabase.testCaseInfo(testCaseName)
440 codeTest = getCodeTest(testCaseName)
442 infoDict = codeTest.getInfoDict()
450 def __verifyTestCase (self, testCaseName, testResultNode, packagesToTest, modulesToTest):
454 testType = infoDict[
'type']
459 mevis.MLAB.log(
"Trying to load test case %s" % testCaseName)
462 mevis.MLAB.log(
"Loading complete")
465 return None,
'Failed'
468 if testType ==
'GenericTestCase':
469 modulesToTest, modulesNotToTest = testCase.getModulesToTest(modulesToTest, packagesToTest)
470 for moduleName
in modulesToTest:
471 etree.SubElement(testResultNode,
"TestCase", name=testCaseName, type=testType, module=moduleName)
472 for moduleName
in modulesNotToTest:
473 etree.SubElement(testResultNode,
"TestCase", name=testCaseName, type=testType, module=moduleName, status=str(Utils.TEST_STATUS_DO_NOT_TEST_MODULE))
475 etree.SubElement(testResultNode,
'TestCase', name=testCaseName, type=testType)
477 return testCase.getTestCaseNode(),
'Ok'
488 def __verifyTestGroups (self, infoDict, testGroupList):
489 testCaseGroupList = infoDict[
'testGroups'].lower()
if 'testGroups' in infoDict
else ""
490 testCaseGroupList = testCaseGroupList.split()
492 if 'disabled' in testCaseGroupList
or (
'needspublicsdk' in testCaseGroupList
and not _isPublicSDK):
494 testGroupList = [tg.lower()
for tg
in testGroupList]
496 if 'automatic' in testGroupList:
497 if 'manual' in testCaseGroupList:
500 for testGroup
in testGroupList:
501 if testGroup
in testCaseGroupList:
514 def __verifyPackages (self, testCfg):
515 packageRoot = testCfg.find(
'Packages')
519 tPkgNode = packageRoot.find(
'Tested')
520 aPkgNode = packageRoot.find(
'Available')
521 iPkgNode = packageRoot.find(
'Ignored')
524 if len(iPkgNode) > 0:
526 ignoredPackageIDs = set()
527 for ignoredPackage
in iPkgNode:
528 ignoredPackageIDs.add(ignoredPackage.get(
'name'))
534 for packageNode
in tPkgNode.findall(
'Package'):
535 if packageNode.get(
'status')
in (
'Ok',
'Added'):
536 tPkgSet.add(packageNode.get(
'name'))
539 for packageNode
in aPkgNode.findall(
'Package'):
540 if packageNode.get(
'status')
in (
'Ok',
'Added'):
541 aPkgSet.add(packageNode.get(
'name'))
544 self.
__testCfg[
'Packages'][
'Tested'] = list(tPkgSet)
545 self.
__testCfg[
'Packages'][
'Available'] = list(tPkgSet.union(aPkgSet))
548 for item
in tPkgSet.difference(aPkgSet):
549 etree.SubElement(aPkgNode,
'Package', name=item, status=
'Ok')
556 def __verifyPackageListNode (self, packageListNode, ignoredPackageIDs=[]):
558 if len(packageListNode) == 0:
559 etree.SubElement(packageListNode,
"Package", name=
"*/*")
561 for packageNode
in packageListNode:
562 packageID = packageNode.get(
'name')
564 pGroup, pName = packageID.split(
"/")
566 packageNode.set(
'status',
'Invalid')
572 for package
in mevis.MLABPackageManager.packages():
573 packageID = package.packageIdentifier()
574 if not packageID
in ignoredPackageIDs:
575 etree.SubElement(packageListNode,
"Package", name=packageID, status=
'Added')
577 packageNode.set(
'status',
'Substituted')
579 packageNode.set(
'status',
'Failed')
581 packageNode.set(
'status',
'Substituted')
582 for package
in mevis.MLABPackageManager.packagesByGroup(pGroup):
583 packageID = package.packageIdentifier()
584 if not packageID
in ignoredPackageIDs:
585 etree.SubElement(packageListNode,
"Package", name=packageID, status=
'Added')
589 if mevis.MLABPackageManager.packageByIdentifier(packageID):
590 packageNode.set(
'status',
'Ok')
592 packageNode.set(
'status',
'Failed')
601 def __collectModulesToTest (self, testCfg):
602 modulesNode = testCfg.find(
'Modules')
606 for filterNode
in modulesNode.findall(
'Filter'):
607 key, valueList = filterNode.text.split(
":")
609 for value
in valueList.split(
","):
610 filterDict[key].append(value)
613 availableModuleList = []
614 for packageName
in self.
__testCfg[
'Packages'][
'Tested']:
615 availableModuleList.extend(mevis.MLAB.allModulesForPackageIdentifier(packageName))
618 if len(modulesNode.findall(
'Module')) == 0:
619 for moduleName
in availableModuleList:
620 etree.SubElement(modulesNode,
"Module", name=moduleName)
623 for moduleNode
in modulesNode.findall(
'Module'):
624 moduleName = moduleNode.get(
'name')
625 moduleInfo = mevis.MLAB.moduleInfo(moduleName)
627 if moduleInfo
and moduleName
in availableModuleList:
631 if not f
in moduleInfo
or not [x
for x
in filterDict[f]
if re.search(x, moduleInfo[f])]:
635 self.
__testCfg[
'Modules'].append(moduleName)
636 moduleNode.set(
'status', str(Utils.TEST_STATUS_OK))
638 moduleNode.set(
'isModuleFiltered', str(
True))
640 modulesNode.set(
'status', str(Utils.TEST_STATUS_ERROR))
645 def __buildModuleInfo (self, modulesInfoNode):
646 for moduleName
in self.
__testCfg[
'Modules']:
647 moduleInfo = mevis.MLAB.moduleInfo(moduleName)
648 newModuleNode = etree.SubElement(modulesInfoNode,
'Module', name=moduleName)
649 for key
in moduleInfo:
650 value = moduleInfo[key]
651 etree.SubElement(newModuleNode, key, type=value.__class__.__name__).text =
unicode(value)
658 def __setPackageList (self, xmlNode, resultNode):
660 for item
in xmlNode.findall(
'Package'):
661 packageList.append(item.get(
'name'))
662 TestHelper.getInstance().setPackageList(packageList)
663 resultNode.set(
'errorCode',
'Ok')
672 resultNode = etree.Element(
'Result')
673 if not TestHelper.getInstance().getGlobalContext():
675 mevis.MLAB.logWarning(
"Test case manager has been closed - aborting test execution")
677 command = requestNode.get(
'type')
678 mevis.MLAB.log(
"Running command '%s'" % command)
681 if command ==
'Quit':
682 resultNode.set(
'errorCode',
'Ok')
684 elif command ==
'DoTest':
685 testCaseNode = requestNode.find(
'TestCase')
686 if self.
__com and testCaseNode.get(
"requestProgress") ==
"1":
689 self.
__doTest(testCaseNode, resultNode)
691 if self.
__com and testCaseNode.get(
"requestProgress") ==
"1":
694 elif command ==
'SetPackageList': self.
__setPackageList(requestNode.find(
'Packages'), resultNode)
695 elif command ==
'BuildTestAgenda': self.
__buildTestAgenda(requestNode.find(
'Configuration'), resultNode)
698 raise Exception(
'Unknown command: %s' % (command))
700 resultNode.set(
'errorCode',
'Exception')
701 etree.SubElement(resultNode,
'Error').text = traceback.format_exc()
702 mevis.MLAB.log(traceback.format_exc())
705 content = etree.tostring(resultNode).decode()
706 mevis.MLAB.log(
"Result content: %s%s" % (content[:150],
" [...]" if len(content)>150
else ""))
716 request = self.
__com.recv(20)
717 if request
in (
'',
None):
718 mevis.MLAB.logError(self.
__com.getLastErrorWithMessage()[1])
721 requestNode = etree.fromstring(request)
724 if not self.
__com.send(etree.tostring(resultNode), 5):
725 mevis.MLAB.logError(
"Couldn't send result to request %s" % (request))
727 self.
__com.disconnect()