plugin.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. import os
  2. import py
  3. import pytest
  4. def auto_detect_cpus():
  5. try:
  6. from os import sched_getaffinity
  7. except ImportError:
  8. if os.environ.get("TRAVIS") == "true":
  9. # workaround https://bitbucket.org/pypy/pypy/issues/2375
  10. return 2
  11. try:
  12. from os import cpu_count
  13. except ImportError:
  14. from multiprocessing import cpu_count
  15. else:
  16. def cpu_count():
  17. return len(sched_getaffinity(0))
  18. try:
  19. n = cpu_count()
  20. except NotImplementedError:
  21. return 1
  22. return n if n else 1
  23. def parse_numprocesses(s):
  24. if s == "auto":
  25. return auto_detect_cpus()
  26. else:
  27. return int(s)
  28. def pytest_addoption(parser):
  29. group = parser.getgroup("xdist", "distributed and subprocess testing")
  30. group._addoption(
  31. "-n",
  32. "--numprocesses",
  33. dest="numprocesses",
  34. metavar="numprocesses",
  35. action="store",
  36. type=parse_numprocesses,
  37. help="shortcut for '--dist=load --tx=NUM*popen', "
  38. "you can use 'auto' here for auto detection CPUs number on "
  39. "host system",
  40. )
  41. group.addoption(
  42. "--max-worker-restart",
  43. "--max-slave-restart",
  44. action="store",
  45. default=None,
  46. dest="maxworkerrestart",
  47. help="maximum number of workers that can be restarted "
  48. "when crashed (set to zero to disable this feature)\n"
  49. "'--max-slave-restart' option is deprecated and will be removed in "
  50. "a future release",
  51. )
  52. group.addoption(
  53. "--dist",
  54. metavar="distmode",
  55. action="store",
  56. choices=["each", "load", "loadscope", "loadfile", "no"],
  57. dest="dist",
  58. default="no",
  59. help=(
  60. "set mode for distributing tests to exec environments.\n\n"
  61. "each: send each test to all available environments.\n\n"
  62. "load: load balance by sending any pending test to any"
  63. " available environment.\n\n"
  64. "loadscope: load balance by sending pending groups of tests in"
  65. " the same scope to any available environment.\n\n"
  66. "loadfile: load balance by sending test grouped by file"
  67. " to any available environment.\n\n"
  68. "(default) no: run tests inprocess, don't distribute."
  69. ),
  70. )
  71. group.addoption(
  72. "--tx",
  73. dest="tx",
  74. action="append",
  75. default=[],
  76. metavar="xspec",
  77. help=(
  78. "add a test execution environment. some examples: "
  79. "--tx popen//python=python2.5 --tx socket=192.168.1.102:8888 "
  80. "--tx ssh=user@codespeak.net//chdir=testcache"
  81. ),
  82. )
  83. group._addoption(
  84. "-d",
  85. action="store_true",
  86. dest="distload",
  87. default=False,
  88. help="load-balance tests. shortcut for '--dist=load'",
  89. )
  90. group.addoption(
  91. "--rsyncdir",
  92. action="append",
  93. default=[],
  94. metavar="DIR",
  95. help="add directory for rsyncing to remote tx nodes.",
  96. )
  97. group.addoption(
  98. "--rsyncignore",
  99. action="append",
  100. default=[],
  101. metavar="GLOB",
  102. help="add expression for ignores when rsyncing to remote tx nodes.",
  103. )
  104. group.addoption(
  105. "--boxed",
  106. action="store_true",
  107. help="backward compatibility alias for pytest-forked --forked",
  108. )
  109. parser.addini(
  110. "rsyncdirs",
  111. "list of (relative) paths to be rsynced for" " remote distributed testing.",
  112. type="pathlist",
  113. )
  114. parser.addini(
  115. "rsyncignore",
  116. "list of (relative) glob-style paths to be ignored " "for rsyncing.",
  117. type="pathlist",
  118. )
  119. parser.addini(
  120. "looponfailroots",
  121. type="pathlist",
  122. help="directories to check for changes",
  123. default=[py.path.local()],
  124. )
  125. # -------------------------------------------------------------------------
  126. # distributed testing hooks
  127. # -------------------------------------------------------------------------
  128. def pytest_addhooks(pluginmanager):
  129. from xdist import newhooks
  130. # avoid warnings with pytest-2.8
  131. method = getattr(pluginmanager, "add_hookspecs", None)
  132. if method is None:
  133. method = pluginmanager.addhooks
  134. method(newhooks)
  135. # -------------------------------------------------------------------------
  136. # distributed testing initialization
  137. # -------------------------------------------------------------------------
  138. @pytest.mark.trylast
  139. def pytest_configure(config):
  140. if config.getoption("dist") != "no" and not config.getvalue("collectonly"):
  141. from xdist.dsession import DSession
  142. session = DSession(config)
  143. config.pluginmanager.register(session, "dsession")
  144. tr = config.pluginmanager.getplugin("terminalreporter")
  145. tr.showfspath = False
  146. if config.getoption("boxed"):
  147. config.option.forked = True
  148. @pytest.mark.tryfirst
  149. def pytest_cmdline_main(config):
  150. if config.option.numprocesses:
  151. if config.option.dist == "no":
  152. config.option.dist = "load"
  153. config.option.tx = ["popen"] * config.option.numprocesses
  154. if config.option.distload:
  155. config.option.dist = "load"
  156. val = config.getvalue
  157. if not val("collectonly"):
  158. usepdb = config.getoption("usepdb") # a core option
  159. if val("dist") != "no":
  160. if usepdb:
  161. raise pytest.UsageError(
  162. "--pdb is incompatible with distributing tests; try using -n0."
  163. ) # noqa: E501
  164. # -------------------------------------------------------------------------
  165. # fixtures
  166. # -------------------------------------------------------------------------
  167. @pytest.fixture(scope="session")
  168. def worker_id(request):
  169. """Return the id of the current worker ('gw0', 'gw1', etc) or 'master'
  170. if running on the master node.
  171. """
  172. if hasattr(request.config, "workerinput"):
  173. return request.config.workerinput["workerid"]
  174. else:
  175. return "master"