junitxml.py 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564
  1. """
  2. report test results in JUnit-XML format,
  3. for use with Jenkins and build integration servers.
  4. Based on initial code from Ross Lawley.
  5. Output conforms to https://github.com/jenkinsci/xunit-plugin/blob/master/
  6. src/main/resources/org/jenkinsci/plugins/xunit/types/model/xsd/junit-10.xsd
  7. """
  8. from __future__ import absolute_import, division, print_function
  9. import functools
  10. import py
  11. import os
  12. import re
  13. import sys
  14. import time
  15. import pytest
  16. from _pytest import nodes
  17. from _pytest.config import filename_arg
  18. # Python 2.X and 3.X compatibility
  19. if sys.version_info[0] < 3:
  20. from codecs import open
  21. else:
  22. unichr = chr
  23. unicode = str
  24. long = int
  25. class Junit(py.xml.Namespace):
  26. pass
  27. # We need to get the subset of the invalid unicode ranges according to
  28. # XML 1.0 which are valid in this python build. Hence we calculate
  29. # this dynamically instead of hardcoding it. The spec range of valid
  30. # chars is: Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD]
  31. # | [#x10000-#x10FFFF]
  32. _legal_chars = (0x09, 0x0A, 0x0d)
  33. _legal_ranges = ((0x20, 0x7E), (0x80, 0xD7FF), (0xE000, 0xFFFD), (0x10000, 0x10FFFF))
  34. _legal_xml_re = [
  35. unicode("%s-%s") % (unichr(low), unichr(high))
  36. for (low, high) in _legal_ranges
  37. if low < sys.maxunicode
  38. ]
  39. _legal_xml_re = [unichr(x) for x in _legal_chars] + _legal_xml_re
  40. illegal_xml_re = re.compile(unicode("[^%s]") % unicode("").join(_legal_xml_re))
  41. del _legal_chars
  42. del _legal_ranges
  43. del _legal_xml_re
  44. _py_ext_re = re.compile(r"\.py$")
  45. def bin_xml_escape(arg):
  46. def repl(matchobj):
  47. i = ord(matchobj.group())
  48. if i <= 0xFF:
  49. return unicode("#x%02X") % i
  50. else:
  51. return unicode("#x%04X") % i
  52. return py.xml.raw(illegal_xml_re.sub(repl, py.xml.escape(arg)))
  53. class _NodeReporter(object):
  54. def __init__(self, nodeid, xml):
  55. self.id = nodeid
  56. self.xml = xml
  57. self.add_stats = self.xml.add_stats
  58. self.duration = 0
  59. self.properties = []
  60. self.nodes = []
  61. self.testcase = None
  62. self.attrs = {}
  63. def append(self, node):
  64. self.xml.add_stats(type(node).__name__)
  65. self.nodes.append(node)
  66. def add_property(self, name, value):
  67. self.properties.append((str(name), bin_xml_escape(value)))
  68. def add_attribute(self, name, value):
  69. self.attrs[str(name)] = bin_xml_escape(value)
  70. def make_properties_node(self):
  71. """Return a Junit node containing custom properties, if any.
  72. """
  73. if self.properties:
  74. return Junit.properties(
  75. [
  76. Junit.property(name=name, value=value)
  77. for name, value in self.properties
  78. ]
  79. )
  80. return ""
  81. def record_testreport(self, testreport):
  82. assert not self.testcase
  83. names = mangle_test_address(testreport.nodeid)
  84. existing_attrs = self.attrs
  85. classnames = names[:-1]
  86. if self.xml.prefix:
  87. classnames.insert(0, self.xml.prefix)
  88. attrs = {
  89. "classname": ".".join(classnames),
  90. "name": bin_xml_escape(names[-1]),
  91. "file": testreport.location[0],
  92. }
  93. if testreport.location[1] is not None:
  94. attrs["line"] = testreport.location[1]
  95. if hasattr(testreport, "url"):
  96. attrs["url"] = testreport.url
  97. self.attrs = attrs
  98. self.attrs.update(existing_attrs) # restore any user-defined attributes
  99. def to_xml(self):
  100. testcase = Junit.testcase(time=self.duration, **self.attrs)
  101. testcase.append(self.make_properties_node())
  102. for node in self.nodes:
  103. testcase.append(node)
  104. return testcase
  105. def _add_simple(self, kind, message, data=None):
  106. data = bin_xml_escape(data)
  107. node = kind(data, message=message)
  108. self.append(node)
  109. def write_captured_output(self, report):
  110. content_out = report.capstdout
  111. content_log = report.caplog
  112. content_err = report.capstderr
  113. if content_log or content_out:
  114. if content_log and self.xml.logging == "system-out":
  115. if content_out:
  116. # syncing stdout and the log-output is not done yet. It's
  117. # probably not worth the effort. Therefore, first the captured
  118. # stdout is shown and then the captured logs.
  119. content = "\n".join(
  120. [
  121. " Captured Stdout ".center(80, "-"),
  122. content_out,
  123. "",
  124. " Captured Log ".center(80, "-"),
  125. content_log,
  126. ]
  127. )
  128. else:
  129. content = content_log
  130. else:
  131. content = content_out
  132. if content:
  133. tag = getattr(Junit, "system-out")
  134. self.append(tag(bin_xml_escape(content)))
  135. if content_log or content_err:
  136. if content_log and self.xml.logging == "system-err":
  137. if content_err:
  138. content = "\n".join(
  139. [
  140. " Captured Stderr ".center(80, "-"),
  141. content_err,
  142. "",
  143. " Captured Log ".center(80, "-"),
  144. content_log,
  145. ]
  146. )
  147. else:
  148. content = content_log
  149. else:
  150. content = content_err
  151. if content:
  152. tag = getattr(Junit, "system-err")
  153. self.append(tag(bin_xml_escape(content)))
  154. def append_pass(self, report):
  155. self.add_stats("passed")
  156. def append_failure(self, report):
  157. # msg = str(report.longrepr.reprtraceback.extraline)
  158. if hasattr(report, "wasxfail"):
  159. self._add_simple(Junit.skipped, "xfail-marked test passes unexpectedly")
  160. else:
  161. if hasattr(report.longrepr, "reprcrash"):
  162. message = report.longrepr.reprcrash.message
  163. elif isinstance(report.longrepr, (unicode, str)):
  164. message = report.longrepr
  165. else:
  166. message = str(report.longrepr)
  167. message = bin_xml_escape(message)
  168. fail = Junit.failure(message=message)
  169. fail.append(bin_xml_escape(report.longrepr))
  170. self.append(fail)
  171. def append_collect_error(self, report):
  172. # msg = str(report.longrepr.reprtraceback.extraline)
  173. self.append(
  174. Junit.error(bin_xml_escape(report.longrepr), message="collection failure")
  175. )
  176. def append_collect_skipped(self, report):
  177. self._add_simple(Junit.skipped, "collection skipped", report.longrepr)
  178. def append_error(self, report):
  179. if getattr(report, "when", None) == "teardown":
  180. msg = "test teardown failure"
  181. else:
  182. msg = "test setup failure"
  183. self._add_simple(Junit.error, msg, report.longrepr)
  184. def append_skipped(self, report):
  185. if hasattr(report, "wasxfail"):
  186. self._add_simple(Junit.skipped, "expected test failure", report.wasxfail)
  187. else:
  188. filename, lineno, skipreason = report.longrepr
  189. if skipreason.startswith("Skipped: "):
  190. skipreason = bin_xml_escape(skipreason[9:])
  191. self.append(
  192. Junit.skipped(
  193. "%s:%s: %s" % (filename, lineno, skipreason),
  194. type="pytest.skip",
  195. message=skipreason,
  196. )
  197. )
  198. self.write_captured_output(report)
  199. def finalize(self):
  200. data = self.to_xml().unicode(indent=0)
  201. self.__dict__.clear()
  202. self.to_xml = lambda: py.xml.raw(data)
  203. @pytest.fixture
  204. def record_property(request):
  205. """Add an extra properties the calling test.
  206. User properties become part of the test report and are available to the
  207. configured reporters, like JUnit XML.
  208. The fixture is callable with ``(name, value)``, with value being automatically
  209. xml-encoded.
  210. Example::
  211. def test_function(record_property):
  212. record_property("example_key", 1)
  213. """
  214. def append_property(name, value):
  215. request.node.user_properties.append((name, value))
  216. return append_property
  217. @pytest.fixture
  218. def record_xml_property(record_property):
  219. """(Deprecated) use record_property."""
  220. import warnings
  221. from _pytest import deprecated
  222. warnings.warn(deprecated.RECORD_XML_PROPERTY, DeprecationWarning, stacklevel=2)
  223. return record_property
  224. @pytest.fixture
  225. def record_xml_attribute(request):
  226. """Add extra xml attributes to the tag for the calling test.
  227. The fixture is callable with ``(name, value)``, with value being
  228. automatically xml-encoded
  229. """
  230. request.node.warn(
  231. code="C3", message="record_xml_attribute is an experimental feature"
  232. )
  233. xml = getattr(request.config, "_xml", None)
  234. if xml is not None:
  235. node_reporter = xml.node_reporter(request.node.nodeid)
  236. return node_reporter.add_attribute
  237. else:
  238. def add_attr_noop(name, value):
  239. pass
  240. return add_attr_noop
  241. def pytest_addoption(parser):
  242. group = parser.getgroup("terminal reporting")
  243. group.addoption(
  244. "--junitxml",
  245. "--junit-xml",
  246. action="store",
  247. dest="xmlpath",
  248. metavar="path",
  249. type=functools.partial(filename_arg, optname="--junitxml"),
  250. default=None,
  251. help="create junit-xml style report file at given path.",
  252. )
  253. group.addoption(
  254. "--junitprefix",
  255. "--junit-prefix",
  256. action="store",
  257. metavar="str",
  258. default=None,
  259. help="prepend prefix to classnames in junit-xml output",
  260. )
  261. parser.addini(
  262. "junit_suite_name", "Test suite name for JUnit report", default="pytest"
  263. )
  264. parser.addini(
  265. "junit_logging",
  266. "Write captured log messages to JUnit report: "
  267. "one of no|system-out|system-err",
  268. default="no",
  269. ) # choices=['no', 'stdout', 'stderr'])
  270. def pytest_configure(config):
  271. xmlpath = config.option.xmlpath
  272. # prevent opening xmllog on slave nodes (xdist)
  273. if xmlpath and not hasattr(config, "slaveinput"):
  274. config._xml = LogXML(
  275. xmlpath,
  276. config.option.junitprefix,
  277. config.getini("junit_suite_name"),
  278. config.getini("junit_logging"),
  279. )
  280. config.pluginmanager.register(config._xml)
  281. def pytest_unconfigure(config):
  282. xml = getattr(config, "_xml", None)
  283. if xml:
  284. del config._xml
  285. config.pluginmanager.unregister(xml)
  286. def mangle_test_address(address):
  287. path, possible_open_bracket, params = address.partition("[")
  288. names = path.split("::")
  289. try:
  290. names.remove("()")
  291. except ValueError:
  292. pass
  293. # convert file path to dotted path
  294. names[0] = names[0].replace(nodes.SEP, ".")
  295. names[0] = _py_ext_re.sub("", names[0])
  296. # put any params back
  297. names[-1] += possible_open_bracket + params
  298. return names
  299. class LogXML(object):
  300. def __init__(self, logfile, prefix, suite_name="pytest", logging="no"):
  301. logfile = os.path.expanduser(os.path.expandvars(logfile))
  302. self.logfile = os.path.normpath(os.path.abspath(logfile))
  303. self.prefix = prefix
  304. self.suite_name = suite_name
  305. self.logging = logging
  306. self.stats = dict.fromkeys(["error", "passed", "failure", "skipped"], 0)
  307. self.node_reporters = {} # nodeid -> _NodeReporter
  308. self.node_reporters_ordered = []
  309. self.global_properties = []
  310. # List of reports that failed on call but teardown is pending.
  311. self.open_reports = []
  312. self.cnt_double_fail_tests = 0
  313. def finalize(self, report):
  314. nodeid = getattr(report, "nodeid", report)
  315. # local hack to handle xdist report order
  316. slavenode = getattr(report, "node", None)
  317. reporter = self.node_reporters.pop((nodeid, slavenode))
  318. if reporter is not None:
  319. reporter.finalize()
  320. def node_reporter(self, report):
  321. nodeid = getattr(report, "nodeid", report)
  322. # local hack to handle xdist report order
  323. slavenode = getattr(report, "node", None)
  324. key = nodeid, slavenode
  325. if key in self.node_reporters:
  326. # TODO: breasks for --dist=each
  327. return self.node_reporters[key]
  328. reporter = _NodeReporter(nodeid, self)
  329. self.node_reporters[key] = reporter
  330. self.node_reporters_ordered.append(reporter)
  331. return reporter
  332. def add_stats(self, key):
  333. if key in self.stats:
  334. self.stats[key] += 1
  335. def _opentestcase(self, report):
  336. reporter = self.node_reporter(report)
  337. reporter.record_testreport(report)
  338. return reporter
  339. def pytest_runtest_logreport(self, report):
  340. """handle a setup/call/teardown report, generating the appropriate
  341. xml tags as necessary.
  342. note: due to plugins like xdist, this hook may be called in interlaced
  343. order with reports from other nodes. for example:
  344. usual call order:
  345. -> setup node1
  346. -> call node1
  347. -> teardown node1
  348. -> setup node2
  349. -> call node2
  350. -> teardown node2
  351. possible call order in xdist:
  352. -> setup node1
  353. -> call node1
  354. -> setup node2
  355. -> call node2
  356. -> teardown node2
  357. -> teardown node1
  358. """
  359. close_report = None
  360. if report.passed:
  361. if report.when == "call": # ignore setup/teardown
  362. reporter = self._opentestcase(report)
  363. reporter.append_pass(report)
  364. elif report.failed:
  365. if report.when == "teardown":
  366. # The following vars are needed when xdist plugin is used
  367. report_wid = getattr(report, "worker_id", None)
  368. report_ii = getattr(report, "item_index", None)
  369. close_report = next(
  370. (
  371. rep
  372. for rep in self.open_reports
  373. if (
  374. rep.nodeid == report.nodeid
  375. and getattr(rep, "item_index", None) == report_ii
  376. and getattr(rep, "worker_id", None) == report_wid
  377. )
  378. ),
  379. None,
  380. )
  381. if close_report:
  382. # We need to open new testcase in case we have failure in
  383. # call and error in teardown in order to follow junit
  384. # schema
  385. self.finalize(close_report)
  386. self.cnt_double_fail_tests += 1
  387. reporter = self._opentestcase(report)
  388. if report.when == "call":
  389. reporter.append_failure(report)
  390. self.open_reports.append(report)
  391. else:
  392. reporter.append_error(report)
  393. elif report.skipped:
  394. reporter = self._opentestcase(report)
  395. reporter.append_skipped(report)
  396. self.update_testcase_duration(report)
  397. if report.when == "teardown":
  398. reporter = self._opentestcase(report)
  399. reporter.write_captured_output(report)
  400. for propname, propvalue in report.user_properties:
  401. reporter.add_property(propname, propvalue)
  402. self.finalize(report)
  403. report_wid = getattr(report, "worker_id", None)
  404. report_ii = getattr(report, "item_index", None)
  405. close_report = next(
  406. (
  407. rep
  408. for rep in self.open_reports
  409. if (
  410. rep.nodeid == report.nodeid
  411. and getattr(rep, "item_index", None) == report_ii
  412. and getattr(rep, "worker_id", None) == report_wid
  413. )
  414. ),
  415. None,
  416. )
  417. if close_report:
  418. self.open_reports.remove(close_report)
  419. def update_testcase_duration(self, report):
  420. """accumulates total duration for nodeid from given report and updates
  421. the Junit.testcase with the new total if already created.
  422. """
  423. reporter = self.node_reporter(report)
  424. reporter.duration += getattr(report, "duration", 0.0)
  425. def pytest_collectreport(self, report):
  426. if not report.passed:
  427. reporter = self._opentestcase(report)
  428. if report.failed:
  429. reporter.append_collect_error(report)
  430. else:
  431. reporter.append_collect_skipped(report)
  432. def pytest_internalerror(self, excrepr):
  433. reporter = self.node_reporter("internal")
  434. reporter.attrs.update(classname="pytest", name="internal")
  435. reporter._add_simple(Junit.error, "internal error", excrepr)
  436. def pytest_sessionstart(self):
  437. self.suite_start_time = time.time()
  438. def pytest_sessionfinish(self):
  439. dirname = os.path.dirname(os.path.abspath(self.logfile))
  440. if not os.path.isdir(dirname):
  441. os.makedirs(dirname)
  442. logfile = open(self.logfile, "w", encoding="utf-8")
  443. suite_stop_time = time.time()
  444. suite_time_delta = suite_stop_time - self.suite_start_time
  445. numtests = (
  446. self.stats["passed"]
  447. + self.stats["failure"]
  448. + self.stats["skipped"]
  449. + self.stats["error"]
  450. - self.cnt_double_fail_tests
  451. )
  452. logfile.write('<?xml version="1.0" encoding="utf-8"?>')
  453. logfile.write(
  454. Junit.testsuite(
  455. self._get_global_properties_node(),
  456. [x.to_xml() for x in self.node_reporters_ordered],
  457. name=self.suite_name,
  458. errors=self.stats["error"],
  459. failures=self.stats["failure"],
  460. skips=self.stats["skipped"],
  461. tests=numtests,
  462. time="%.3f" % suite_time_delta,
  463. ).unicode(indent=0)
  464. )
  465. logfile.close()
  466. def pytest_terminal_summary(self, terminalreporter):
  467. terminalreporter.write_sep("-", "generated xml file: %s" % (self.logfile))
  468. def add_global_property(self, name, value):
  469. self.global_properties.append((str(name), bin_xml_escape(value)))
  470. def _get_global_properties_node(self):
  471. """Return a Junit node containing custom properties, if any.
  472. """
  473. if self.global_properties:
  474. return Junit.properties(
  475. [
  476. Junit.property(name=name, value=value)
  477. for name, value in self.global_properties
  478. ]
  479. )
  480. return ""