testresult.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. '''Test runner and result class for the regression test suite.
  2. '''
  3. import functools
  4. import io
  5. import sys
  6. import time
  7. import traceback
  8. import unittest
  9. class RegressionTestResult(unittest.TextTestResult):
  10. USE_XML = False
  11. def __init__(self, stream, descriptions, verbosity):
  12. super().__init__(stream=stream, descriptions=descriptions,
  13. verbosity=2 if verbosity else 0)
  14. self.buffer = True
  15. if self.USE_XML:
  16. from xml.etree import ElementTree as ET
  17. from datetime import datetime
  18. self.__ET = ET
  19. self.__suite = ET.Element('testsuite')
  20. self.__suite.set('start', datetime.utcnow().isoformat(' '))
  21. self.__e = None
  22. self.__start_time = None
  23. @classmethod
  24. def __getId(cls, test):
  25. try:
  26. test_id = test.id
  27. except AttributeError:
  28. return str(test)
  29. try:
  30. return test_id()
  31. except TypeError:
  32. return str(test_id)
  33. return repr(test)
  34. def startTest(self, test):
  35. super().startTest(test)
  36. if self.USE_XML:
  37. self.__e = e = self.__ET.SubElement(self.__suite, 'testcase')
  38. self.__start_time = time.perf_counter()
  39. def _add_result(self, test, capture=False, **args):
  40. if not self.USE_XML:
  41. return
  42. e = self.__e
  43. self.__e = None
  44. if e is None:
  45. return
  46. ET = self.__ET
  47. e.set('name', args.pop('name', self.__getId(test)))
  48. e.set('status', args.pop('status', 'run'))
  49. e.set('result', args.pop('result', 'completed'))
  50. if self.__start_time:
  51. e.set('time', f'{time.perf_counter() - self.__start_time:0.6f}')
  52. if capture:
  53. if self._stdout_buffer is not None:
  54. stdout = self._stdout_buffer.getvalue().rstrip()
  55. ET.SubElement(e, 'system-out').text = stdout
  56. if self._stderr_buffer is not None:
  57. stderr = self._stderr_buffer.getvalue().rstrip()
  58. ET.SubElement(e, 'system-err').text = stderr
  59. for k, v in args.items():
  60. if not k or not v:
  61. continue
  62. e2 = ET.SubElement(e, k)
  63. if hasattr(v, 'items'):
  64. for k2, v2 in v.items():
  65. if k2:
  66. e2.set(k2, str(v2))
  67. else:
  68. e2.text = str(v2)
  69. else:
  70. e2.text = str(v)
  71. @classmethod
  72. def __makeErrorDict(cls, err_type, err_value, err_tb):
  73. if isinstance(err_type, type):
  74. if err_type.__module__ == 'builtins':
  75. typename = err_type.__name__
  76. else:
  77. typename = f'{err_type.__module__}.{err_type.__name__}'
  78. else:
  79. typename = repr(err_type)
  80. msg = traceback.format_exception(err_type, err_value, None)
  81. tb = traceback.format_exception(err_type, err_value, err_tb)
  82. return {
  83. 'type': typename,
  84. 'message': ''.join(msg),
  85. '': ''.join(tb),
  86. }
  87. def addError(self, test, err):
  88. self._add_result(test, True, error=self.__makeErrorDict(*err))
  89. super().addError(test, err)
  90. def addExpectedFailure(self, test, err):
  91. self._add_result(test, True, output=self.__makeErrorDict(*err))
  92. super().addExpectedFailure(test, err)
  93. def addFailure(self, test, err):
  94. self._add_result(test, True, failure=self.__makeErrorDict(*err))
  95. super().addFailure(test, err)
  96. def addSkip(self, test, reason):
  97. self._add_result(test, skipped=reason)
  98. super().addSkip(test, reason)
  99. def addSuccess(self, test):
  100. self._add_result(test)
  101. super().addSuccess(test)
  102. def addUnexpectedSuccess(self, test):
  103. self._add_result(test, outcome='UNEXPECTED_SUCCESS')
  104. super().addUnexpectedSuccess(test)
  105. def get_xml_element(self):
  106. if not self.USE_XML:
  107. raise ValueError("USE_XML is false")
  108. e = self.__suite
  109. e.set('tests', str(self.testsRun))
  110. e.set('errors', str(len(self.errors)))
  111. e.set('failures', str(len(self.failures)))
  112. return e
  113. class QuietRegressionTestRunner:
  114. def __init__(self, stream, buffer=False):
  115. self.result = RegressionTestResult(stream, None, 0)
  116. self.result.buffer = buffer
  117. def run(self, test):
  118. test(self.result)
  119. return self.result
  120. def get_test_runner_class(verbosity, buffer=False):
  121. if verbosity:
  122. return functools.partial(unittest.TextTestRunner,
  123. resultclass=RegressionTestResult,
  124. buffer=buffer,
  125. verbosity=verbosity)
  126. return functools.partial(QuietRegressionTestRunner, buffer=buffer)
  127. def get_test_runner(stream, verbosity, capture_output=False):
  128. return get_test_runner_class(verbosity, capture_output)(stream)
  129. if __name__ == '__main__':
  130. import xml.etree.ElementTree as ET
  131. RegressionTestResult.USE_XML = True
  132. class TestTests(unittest.TestCase):
  133. def test_pass(self):
  134. pass
  135. def test_pass_slow(self):
  136. time.sleep(1.0)
  137. def test_fail(self):
  138. print('stdout', file=sys.stdout)
  139. print('stderr', file=sys.stderr)
  140. self.fail('failure message')
  141. def test_error(self):
  142. print('stdout', file=sys.stdout)
  143. print('stderr', file=sys.stderr)
  144. raise RuntimeError('error message')
  145. suite = unittest.TestSuite()
  146. suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestTests))
  147. stream = io.StringIO()
  148. runner_cls = get_test_runner_class(sum(a == '-v' for a in sys.argv))
  149. runner = runner_cls(sys.stdout)
  150. result = runner.run(suite)
  151. print('Output:', stream.getvalue())
  152. print('XML: ', end='')
  153. for s in ET.tostringlist(result.get_xml_element()):
  154. print(s.decode(), end='')
  155. print()