__init__.py 51 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639
  1. # -*- coding: utf-8 -*-
  2. # Copyright (c) 2009, Giampaolo Rodola'. All rights reserved.
  3. # Use of this source code is governed by a BSD-style license that can be
  4. # found in the LICENSE file.
  5. """
  6. Test utilities.
  7. """
  8. from __future__ import print_function
  9. import atexit
  10. import contextlib
  11. import ctypes
  12. import errno
  13. import functools
  14. import gc
  15. import inspect
  16. import os
  17. import random
  18. import re
  19. import select
  20. import shutil
  21. import signal
  22. import socket
  23. import stat
  24. import subprocess
  25. import sys
  26. import tempfile
  27. import textwrap
  28. import threading
  29. import time
  30. import warnings
  31. from socket import AF_INET
  32. from socket import AF_INET6
  33. from socket import SOCK_STREAM
  34. import psutil
  35. from psutil import AIX
  36. from psutil import LINUX
  37. from psutil import MACOS
  38. from psutil import POSIX
  39. from psutil import SUNOS
  40. from psutil import WINDOWS
  41. from psutil._common import bytes2human
  42. from psutil._common import print_color
  43. from psutil._common import supports_ipv6
  44. from psutil._compat import FileExistsError
  45. from psutil._compat import FileNotFoundError
  46. from psutil._compat import PY3
  47. from psutil._compat import range
  48. from psutil._compat import super
  49. from psutil._compat import u
  50. from psutil._compat import unicode
  51. from psutil._compat import which
  52. if PY3:
  53. import unittest
  54. else:
  55. import unittest2 as unittest # requires "pip install unittest2"
  56. try:
  57. from unittest import mock # py3
  58. except ImportError:
  59. with warnings.catch_warnings():
  60. warnings.simplefilter("ignore")
  61. import mock # NOQA - requires "pip install mock"
  62. if sys.version_info >= (3, 4):
  63. import enum
  64. else:
  65. enum = None
  66. __all__ = [
  67. # constants
  68. 'APPVEYOR', 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES',
  69. 'PYPY', 'PYTHON_EXE', 'ROOT_DIR', 'SCRIPTS_DIR', 'TESTFN_PREFIX',
  70. 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', 'TRAVIS', 'CIRRUS',
  71. 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT',
  72. "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS",
  73. "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT",
  74. "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS",
  75. "HAS_SENSORS_TEMPERATURES", "HAS_MEMORY_FULL_INFO",
  76. # subprocesses
  77. 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie',
  78. 'spawn_children_pair',
  79. # threads
  80. 'ThreadTask'
  81. # test utils
  82. 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented',
  83. 'retry_on_failure', 'TestMemoryLeak', 'PsutilTestCase',
  84. 'process_namespace', 'system_namespace', 'print_sysinfo',
  85. # install utils
  86. 'install_pip', 'install_test_deps',
  87. # fs utils
  88. 'chdir', 'safe_rmpath', 'create_exe', 'decode_path', 'encode_path',
  89. 'get_testfn',
  90. # os
  91. 'get_winver', 'get_kernel_version',
  92. # sync primitives
  93. 'call_until', 'wait_for_pid', 'wait_for_file',
  94. # network
  95. 'check_net_address',
  96. 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair',
  97. 'unix_socketpair', 'create_sockets',
  98. # compat
  99. 'reload_module', 'import_module_by_path',
  100. # others
  101. 'warn', 'copyload_shared_lib', 'is_namedtuple',
  102. ]
  103. # ===================================================================
  104. # --- constants
  105. # ===================================================================
  106. # --- platforms
  107. PYPY = '__pypy__' in sys.builtin_module_names
  108. # whether we're running this test suite on a Continuous Integration service
  109. TRAVIS = 'TRAVIS' in os.environ
  110. APPVEYOR = 'APPVEYOR' in os.environ
  111. CIRRUS = 'CIRRUS' in os.environ
  112. GITHUB_WHEELS = 'CIBUILDWHEEL' in os.environ
  113. CI_TESTING = TRAVIS or APPVEYOR or CIRRUS or GITHUB_WHEELS
  114. # are we a 64 bit process?
  115. IS_64BIT = sys.maxsize > 2 ** 32
  116. # --- configurable defaults
  117. # how many times retry_on_failure() decorator will retry
  118. NO_RETRIES = 10
  119. # bytes tolerance for system-wide related tests
  120. TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB
  121. TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB
  122. # the timeout used in functions which have to wait
  123. GLOBAL_TIMEOUT = 5
  124. # be more tolerant if we're on travis / appveyor in order to avoid
  125. # false positives
  126. if CI_TESTING:
  127. NO_RETRIES *= 3
  128. GLOBAL_TIMEOUT *= 3
  129. TOLERANCE_SYS_MEM *= 3
  130. TOLERANCE_DISK_USAGE *= 3
  131. # --- file names
  132. # Disambiguate TESTFN for parallel testing.
  133. if os.name == 'java':
  134. # Jython disallows @ in module names
  135. TESTFN_PREFIX = '$psutil-%s-' % os.getpid()
  136. else:
  137. TESTFN_PREFIX = '@psutil-%s-' % os.getpid()
  138. UNICODE_SUFFIX = u("-ƒőő")
  139. # An invalid unicode string.
  140. if PY3:
  141. INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape')
  142. else:
  143. INVALID_UNICODE_SUFFIX = "f\xc0\x80"
  144. ASCII_FS = sys.getfilesystemencoding().lower() in ('ascii', 'us-ascii')
  145. # --- paths
  146. ROOT_DIR = os.path.realpath(
  147. os.path.join(os.path.dirname(__file__), '..', '..'))
  148. SCRIPTS_DIR = os.path.join(ROOT_DIR, 'scripts')
  149. HERE = os.path.realpath(os.path.dirname(__file__))
  150. # --- support
  151. HAS_CONNECTIONS_UNIX = POSIX and not SUNOS
  152. HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity")
  153. HAS_CPU_FREQ = hasattr(psutil, "cpu_freq")
  154. HAS_GETLOADAVG = hasattr(psutil, "getloadavg")
  155. HAS_ENVIRON = hasattr(psutil.Process, "environ")
  156. HAS_IONICE = hasattr(psutil.Process, "ionice")
  157. HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps")
  158. HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters")
  159. HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num")
  160. HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters")
  161. HAS_RLIMIT = hasattr(psutil.Process, "rlimit")
  162. HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery")
  163. try:
  164. HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery())
  165. except Exception:
  166. HAS_BATTERY = False
  167. HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans")
  168. HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures")
  169. HAS_THREADS = hasattr(psutil.Process, "threads")
  170. SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0
  171. # --- misc
  172. def _get_py_exe():
  173. def attempt(exe):
  174. try:
  175. subprocess.check_call(
  176. [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
  177. except Exception:
  178. return None
  179. else:
  180. return exe
  181. if GITHUB_WHEELS:
  182. if PYPY:
  183. return which("pypy3") if PY3 else which("pypy")
  184. else:
  185. return which('python')
  186. elif MACOS:
  187. exe = \
  188. attempt(sys.executable) or \
  189. attempt(os.path.realpath(sys.executable)) or \
  190. attempt(which("python%s.%s" % sys.version_info[:2])) or \
  191. attempt(psutil.Process().exe())
  192. if not exe:
  193. raise ValueError("can't find python exe real abspath")
  194. return exe
  195. else:
  196. exe = os.path.realpath(sys.executable)
  197. assert os.path.exists(exe), exe
  198. return exe
  199. PYTHON_EXE = _get_py_exe()
  200. DEVNULL = open(os.devnull, 'r+')
  201. atexit.register(DEVNULL.close)
  202. VALID_PROC_STATUSES = [getattr(psutil, x) for x in dir(psutil)
  203. if x.startswith('STATUS_')]
  204. AF_UNIX = getattr(socket, "AF_UNIX", object())
  205. _subprocesses_started = set()
  206. _pids_started = set()
  207. # ===================================================================
  208. # --- threads
  209. # ===================================================================
  210. class ThreadTask(threading.Thread):
  211. """A thread task which does nothing expect staying alive."""
  212. def __init__(self):
  213. super().__init__()
  214. self._running = False
  215. self._interval = 0.001
  216. self._flag = threading.Event()
  217. def __repr__(self):
  218. name = self.__class__.__name__
  219. return '<%s running=%s at %#x>' % (name, self._running, id(self))
  220. def __enter__(self):
  221. self.start()
  222. return self
  223. def __exit__(self, *args, **kwargs):
  224. self.stop()
  225. def start(self):
  226. """Start thread and keep it running until an explicit
  227. stop() request. Polls for shutdown every 'timeout' seconds.
  228. """
  229. if self._running:
  230. raise ValueError("already started")
  231. threading.Thread.start(self)
  232. self._flag.wait()
  233. def run(self):
  234. self._running = True
  235. self._flag.set()
  236. while self._running:
  237. time.sleep(self._interval)
  238. def stop(self):
  239. """Stop thread execution and and waits until it is stopped."""
  240. if not self._running:
  241. raise ValueError("already stopped")
  242. self._running = False
  243. self.join()
  244. # ===================================================================
  245. # --- subprocesses
  246. # ===================================================================
  247. def _reap_children_on_err(fun):
  248. @functools.wraps(fun)
  249. def wrapper(*args, **kwargs):
  250. try:
  251. return fun(*args, **kwargs)
  252. except Exception:
  253. reap_children()
  254. raise
  255. return wrapper
  256. @_reap_children_on_err
  257. def spawn_testproc(cmd=None, **kwds):
  258. """Creates a python subprocess which does nothing for 60 secs and
  259. return it as a subprocess.Popen instance.
  260. If "cmd" is specified that is used instead of python.
  261. By default stdin and stdout are redirected to /dev/null.
  262. It also attemps to make sure the process is in a reasonably
  263. initialized state.
  264. The process is registered for cleanup on reap_children().
  265. """
  266. kwds.setdefault("stdin", DEVNULL)
  267. kwds.setdefault("stdout", DEVNULL)
  268. kwds.setdefault("cwd", os.getcwd())
  269. kwds.setdefault("env", os.environ)
  270. if WINDOWS:
  271. # Prevents the subprocess to open error dialogs. This will also
  272. # cause stderr to be suppressed, which is suboptimal in order
  273. # to debug broken tests.
  274. CREATE_NO_WINDOW = 0x8000000
  275. kwds.setdefault("creationflags", CREATE_NO_WINDOW)
  276. if cmd is None:
  277. testfn = get_testfn()
  278. try:
  279. safe_rmpath(testfn)
  280. pyline = "from time import sleep;" \
  281. "open(r'%s', 'w').close();" \
  282. "sleep(60);" % testfn
  283. cmd = [PYTHON_EXE, "-c", pyline]
  284. sproc = subprocess.Popen(cmd, **kwds)
  285. _subprocesses_started.add(sproc)
  286. wait_for_file(testfn, delete=True, empty=True)
  287. finally:
  288. safe_rmpath(testfn)
  289. else:
  290. sproc = subprocess.Popen(cmd, **kwds)
  291. _subprocesses_started.add(sproc)
  292. wait_for_pid(sproc.pid)
  293. return sproc
  294. @_reap_children_on_err
  295. def spawn_children_pair():
  296. """Create a subprocess which creates another one as in:
  297. A (us) -> B (child) -> C (grandchild).
  298. Return a (child, grandchild) tuple.
  299. The 2 processes are fully initialized and will live for 60 secs
  300. and are registered for cleanup on reap_children().
  301. """
  302. tfile = None
  303. testfn = get_testfn(dir=os.getcwd())
  304. try:
  305. s = textwrap.dedent("""\
  306. import subprocess, os, sys, time
  307. s = "import os, time;"
  308. s += "f = open('%s', 'w');"
  309. s += "f.write(str(os.getpid()));"
  310. s += "f.close();"
  311. s += "time.sleep(60);"
  312. p = subprocess.Popen([r'%s', '-c', s])
  313. p.wait()
  314. """ % (os.path.basename(testfn), PYTHON_EXE))
  315. # On Windows if we create a subprocess with CREATE_NO_WINDOW flag
  316. # set (which is the default) a "conhost.exe" extra process will be
  317. # spawned as a child. We don't want that.
  318. if WINDOWS:
  319. subp, tfile = pyrun(s, creationflags=0)
  320. else:
  321. subp, tfile = pyrun(s)
  322. child = psutil.Process(subp.pid)
  323. grandchild_pid = int(wait_for_file(testfn, delete=True, empty=False))
  324. _pids_started.add(grandchild_pid)
  325. grandchild = psutil.Process(grandchild_pid)
  326. return (child, grandchild)
  327. finally:
  328. safe_rmpath(testfn)
  329. if tfile is not None:
  330. safe_rmpath(tfile)
  331. def spawn_zombie():
  332. """Create a zombie process and return a (parent, zombie) process tuple.
  333. In order to kill the zombie parent must be terminate()d first, then
  334. zombie must be wait()ed on.
  335. """
  336. assert psutil.POSIX
  337. unix_file = get_testfn()
  338. src = textwrap.dedent("""\
  339. import os, sys, time, socket, contextlib
  340. child_pid = os.fork()
  341. if child_pid > 0:
  342. time.sleep(3000)
  343. else:
  344. # this is the zombie process
  345. s = socket.socket(socket.AF_UNIX)
  346. with contextlib.closing(s):
  347. s.connect('%s')
  348. if sys.version_info < (3, ):
  349. pid = str(os.getpid())
  350. else:
  351. pid = bytes(str(os.getpid()), 'ascii')
  352. s.sendall(pid)
  353. """ % unix_file)
  354. tfile = None
  355. sock = bind_unix_socket(unix_file)
  356. try:
  357. sock.settimeout(GLOBAL_TIMEOUT)
  358. parent, tfile = pyrun(src)
  359. conn, _ = sock.accept()
  360. try:
  361. select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT)
  362. zpid = int(conn.recv(1024))
  363. _pids_started.add(zpid)
  364. zombie = psutil.Process(zpid)
  365. call_until(lambda: zombie.status(), "ret == psutil.STATUS_ZOMBIE")
  366. return (parent, zombie)
  367. finally:
  368. conn.close()
  369. finally:
  370. sock.close()
  371. safe_rmpath(unix_file)
  372. if tfile is not None:
  373. safe_rmpath(tfile)
  374. @_reap_children_on_err
  375. def pyrun(src, **kwds):
  376. """Run python 'src' code string in a separate interpreter.
  377. Returns a subprocess.Popen instance and the test file where the source
  378. code was written.
  379. """
  380. kwds.setdefault("stdout", None)
  381. kwds.setdefault("stderr", None)
  382. srcfile = get_testfn()
  383. try:
  384. with open(srcfile, 'wt') as f:
  385. f.write(src)
  386. subp = spawn_testproc([PYTHON_EXE, f.name], **kwds)
  387. wait_for_pid(subp.pid)
  388. return (subp, srcfile)
  389. except Exception:
  390. safe_rmpath(srcfile)
  391. raise
  392. @_reap_children_on_err
  393. def sh(cmd, **kwds):
  394. """run cmd in a subprocess and return its output.
  395. raises RuntimeError on error.
  396. """
  397. shell = True if isinstance(cmd, (str, unicode)) else False
  398. # Prevents subprocess to open error dialogs in case of error.
  399. flags = 0x8000000 if WINDOWS and shell else 0
  400. kwds.setdefault("shell", shell)
  401. kwds.setdefault("stdout", subprocess.PIPE)
  402. kwds.setdefault("stderr", subprocess.PIPE)
  403. kwds.setdefault("universal_newlines", True)
  404. kwds.setdefault("creationflags", flags)
  405. p = subprocess.Popen(cmd, **kwds)
  406. _subprocesses_started.add(p)
  407. if PY3:
  408. stdout, stderr = p.communicate(timeout=GLOBAL_TIMEOUT)
  409. else:
  410. stdout, stderr = p.communicate()
  411. if p.returncode != 0:
  412. raise RuntimeError(stderr)
  413. if stderr:
  414. warn(stderr)
  415. if stdout.endswith('\n'):
  416. stdout = stdout[:-1]
  417. return stdout
  418. def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT):
  419. """Terminate a process and wait() for it.
  420. Process can be a PID or an instance of psutil.Process(),
  421. subprocess.Popen() or psutil.Popen().
  422. If it's a subprocess.Popen() or psutil.Popen() instance also closes
  423. its stdin / stdout / stderr fds.
  424. PID is wait()ed even if the process is already gone (kills zombies).
  425. Does nothing if the process does not exist.
  426. Return process exit status.
  427. """
  428. if POSIX:
  429. from psutil._psposix import wait_pid
  430. def wait(proc, timeout):
  431. if isinstance(proc, subprocess.Popen) and not PY3:
  432. proc.wait()
  433. else:
  434. proc.wait(timeout)
  435. if WINDOWS and isinstance(proc, subprocess.Popen):
  436. # Otherwise PID may still hang around.
  437. try:
  438. return psutil.Process(proc.pid).wait(timeout)
  439. except psutil.NoSuchProcess:
  440. pass
  441. def sendsig(proc, sig):
  442. # XXX: otherwise the build hangs for some reason.
  443. if MACOS and GITHUB_WHEELS:
  444. sig = signal.SIGKILL
  445. # If the process received SIGSTOP, SIGCONT is necessary first,
  446. # otherwise SIGTERM won't work.
  447. if POSIX and sig != signal.SIGKILL:
  448. proc.send_signal(signal.SIGCONT)
  449. proc.send_signal(sig)
  450. def term_subproc(proc, timeout):
  451. try:
  452. sendsig(proc, sig)
  453. except OSError as err:
  454. if WINDOWS and err.winerror == 6: # "invalid handle"
  455. pass
  456. elif err.errno != errno.ESRCH:
  457. raise
  458. return wait(proc, timeout)
  459. def term_psproc(proc, timeout):
  460. try:
  461. sendsig(proc, sig)
  462. except psutil.NoSuchProcess:
  463. pass
  464. return wait(proc, timeout)
  465. def term_pid(pid, timeout):
  466. try:
  467. proc = psutil.Process(pid)
  468. except psutil.NoSuchProcess:
  469. # Needed to kill zombies.
  470. if POSIX:
  471. return wait_pid(pid, timeout)
  472. else:
  473. return term_psproc(proc, timeout)
  474. def flush_popen(proc):
  475. if proc.stdout:
  476. proc.stdout.close()
  477. if proc.stderr:
  478. proc.stderr.close()
  479. # Flushing a BufferedWriter may raise an error.
  480. if proc.stdin:
  481. proc.stdin.close()
  482. p = proc_or_pid
  483. try:
  484. if isinstance(p, int):
  485. return term_pid(p, wait_timeout)
  486. elif isinstance(p, (psutil.Process, psutil.Popen)):
  487. return term_psproc(p, wait_timeout)
  488. elif isinstance(p, subprocess.Popen):
  489. return term_subproc(p, wait_timeout)
  490. else:
  491. raise TypeError("wrong type %r" % p)
  492. finally:
  493. if isinstance(p, (subprocess.Popen, psutil.Popen)):
  494. flush_popen(p)
  495. pid = p if isinstance(p, int) else p.pid
  496. assert not psutil.pid_exists(pid), pid
  497. def reap_children(recursive=False):
  498. """Terminate and wait() any subprocess started by this test suite
  499. and any children currently running, ensuring that no processes stick
  500. around to hog resources.
  501. If resursive is True it also tries to terminate and wait()
  502. all grandchildren started by this process.
  503. """
  504. # Get the children here before terminating them, as in case of
  505. # recursive=True we don't want to lose the intermediate reference
  506. # pointing to the grandchildren.
  507. children = psutil.Process().children(recursive=recursive)
  508. # Terminate subprocess.Popen.
  509. while _subprocesses_started:
  510. subp = _subprocesses_started.pop()
  511. terminate(subp)
  512. # Collect started pids.
  513. while _pids_started:
  514. pid = _pids_started.pop()
  515. terminate(pid)
  516. # Terminate children.
  517. if children:
  518. for p in children:
  519. terminate(p, wait_timeout=None)
  520. gone, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT)
  521. for p in alive:
  522. warn("couldn't terminate process %r; attempting kill()" % p)
  523. terminate(p, sig=signal.SIGKILL)
  524. # ===================================================================
  525. # --- OS
  526. # ===================================================================
  527. def get_kernel_version():
  528. """Return a tuple such as (2, 6, 36)."""
  529. if not POSIX:
  530. raise NotImplementedError("not POSIX")
  531. s = ""
  532. uname = os.uname()[2]
  533. for c in uname:
  534. if c.isdigit() or c == '.':
  535. s += c
  536. else:
  537. break
  538. if not s:
  539. raise ValueError("can't parse %r" % uname)
  540. minor = 0
  541. micro = 0
  542. nums = s.split('.')
  543. major = int(nums[0])
  544. if len(nums) >= 2:
  545. minor = int(nums[1])
  546. if len(nums) >= 3:
  547. micro = int(nums[2])
  548. return (major, minor, micro)
  549. def get_winver():
  550. if not WINDOWS:
  551. raise NotImplementedError("not WINDOWS")
  552. wv = sys.getwindowsversion()
  553. if hasattr(wv, 'service_pack_major'): # python >= 2.7
  554. sp = wv.service_pack_major or 0
  555. else:
  556. r = re.search(r"\s\d$", wv[4])
  557. if r:
  558. sp = int(r.group(0))
  559. else:
  560. sp = 0
  561. return (wv[0], wv[1], sp)
  562. # ===================================================================
  563. # --- sync primitives
  564. # ===================================================================
  565. class retry(object):
  566. """A retry decorator."""
  567. def __init__(self,
  568. exception=Exception,
  569. timeout=None,
  570. retries=None,
  571. interval=0.001,
  572. logfun=None,
  573. ):
  574. if timeout and retries:
  575. raise ValueError("timeout and retries args are mutually exclusive")
  576. self.exception = exception
  577. self.timeout = timeout
  578. self.retries = retries
  579. self.interval = interval
  580. self.logfun = logfun
  581. def __iter__(self):
  582. if self.timeout:
  583. stop_at = time.time() + self.timeout
  584. while time.time() < stop_at:
  585. yield
  586. elif self.retries:
  587. for _ in range(self.retries):
  588. yield
  589. else:
  590. while True:
  591. yield
  592. def sleep(self):
  593. if self.interval is not None:
  594. time.sleep(self.interval)
  595. def __call__(self, fun):
  596. @functools.wraps(fun)
  597. def wrapper(*args, **kwargs):
  598. exc = None
  599. for _ in self:
  600. try:
  601. return fun(*args, **kwargs)
  602. except self.exception as _: # NOQA
  603. exc = _
  604. if self.logfun is not None:
  605. self.logfun(exc)
  606. self.sleep()
  607. continue
  608. if PY3:
  609. raise exc
  610. else:
  611. raise
  612. # This way the user of the decorated function can change config
  613. # parameters.
  614. wrapper.decorator = self
  615. return wrapper
  616. @retry(exception=psutil.NoSuchProcess, logfun=None, timeout=GLOBAL_TIMEOUT,
  617. interval=0.001)
  618. def wait_for_pid(pid):
  619. """Wait for pid to show up in the process list then return.
  620. Used in the test suite to give time the sub process to initialize.
  621. """
  622. psutil.Process(pid)
  623. if WINDOWS:
  624. # give it some more time to allow better initialization
  625. time.sleep(0.01)
  626. @retry(exception=(FileNotFoundError, AssertionError), logfun=None,
  627. timeout=GLOBAL_TIMEOUT, interval=0.001)
  628. def wait_for_file(fname, delete=True, empty=False):
  629. """Wait for a file to be written on disk with some content."""
  630. with open(fname, "rb") as f:
  631. data = f.read()
  632. if not empty:
  633. assert data
  634. if delete:
  635. safe_rmpath(fname)
  636. return data
  637. @retry(exception=AssertionError, logfun=None, timeout=GLOBAL_TIMEOUT,
  638. interval=0.001)
  639. def call_until(fun, expr):
  640. """Keep calling function for timeout secs and exit if eval()
  641. expression is True.
  642. """
  643. ret = fun()
  644. assert eval(expr)
  645. return ret
  646. # ===================================================================
  647. # --- fs
  648. # ===================================================================
  649. def safe_rmpath(path):
  650. "Convenience function for removing temporary test files or dirs"
  651. def retry_fun(fun):
  652. # On Windows it could happen that the file or directory has
  653. # open handles or references preventing the delete operation
  654. # to succeed immediately, so we retry for a while. See:
  655. # https://bugs.python.org/issue33240
  656. stop_at = time.time() + GLOBAL_TIMEOUT
  657. while time.time() < stop_at:
  658. try:
  659. return fun()
  660. except FileNotFoundError:
  661. pass
  662. except WindowsError as _:
  663. err = _
  664. warn("ignoring %s" % (str(err)))
  665. time.sleep(0.01)
  666. raise err
  667. try:
  668. st = os.stat(path)
  669. if stat.S_ISDIR(st.st_mode):
  670. fun = functools.partial(shutil.rmtree, path)
  671. else:
  672. fun = functools.partial(os.remove, path)
  673. if POSIX:
  674. fun()
  675. else:
  676. retry_fun(fun)
  677. except FileNotFoundError:
  678. pass
  679. def safe_mkdir(dir):
  680. "Convenience function for creating a directory"
  681. try:
  682. os.mkdir(dir)
  683. except FileExistsError:
  684. pass
  685. @contextlib.contextmanager
  686. def chdir(dirname):
  687. "Context manager which temporarily changes the current directory."
  688. curdir = os.getcwd()
  689. try:
  690. os.chdir(dirname)
  691. yield
  692. finally:
  693. os.chdir(curdir)
  694. def create_exe(outpath, c_code=None):
  695. """Creates an executable file in the given location."""
  696. assert not os.path.exists(outpath), outpath
  697. if c_code:
  698. if not which("gcc"):
  699. raise ValueError("gcc is not installed")
  700. if isinstance(c_code, bool): # c_code is True
  701. c_code = textwrap.dedent(
  702. """
  703. #include <unistd.h>
  704. int main() {
  705. pause();
  706. return 1;
  707. }
  708. """)
  709. assert isinstance(c_code, str), c_code
  710. with open(get_testfn(suffix='.c'), 'wt') as f:
  711. f.write(c_code)
  712. try:
  713. subprocess.check_call(["gcc", f.name, "-o", outpath])
  714. finally:
  715. safe_rmpath(f.name)
  716. else:
  717. # copy python executable
  718. shutil.copyfile(PYTHON_EXE, outpath)
  719. if POSIX:
  720. st = os.stat(outpath)
  721. os.chmod(outpath, st.st_mode | stat.S_IEXEC)
  722. def get_testfn(suffix="", dir=None):
  723. """Return an absolute pathname of a file or dir that did not
  724. exist at the time this call is made. Also schedule it for safe
  725. deletion at interpreter exit. It's technically racy but probably
  726. not really due to the time variant.
  727. """
  728. while True:
  729. name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir)
  730. if not os.path.exists(name): # also include dirs
  731. return os.path.realpath(name) # needed for OSX
  732. # ===================================================================
  733. # --- testing
  734. # ===================================================================
  735. class TestCase(unittest.TestCase):
  736. # Print a full path representation of the single unit tests
  737. # being run.
  738. def __str__(self):
  739. fqmod = self.__class__.__module__
  740. if not fqmod.startswith('psutil.'):
  741. fqmod = 'psutil.tests.' + fqmod
  742. return "%s.%s.%s" % (
  743. fqmod, self.__class__.__name__, self._testMethodName)
  744. # assertRaisesRegexp renamed to assertRaisesRegex in 3.3;
  745. # add support for the new name.
  746. if not hasattr(unittest.TestCase, 'assertRaisesRegex'):
  747. assertRaisesRegex = unittest.TestCase.assertRaisesRegexp
  748. # ...otherwise multiprocessing.Pool complains
  749. if not PY3:
  750. def runTest(self):
  751. pass
  752. # monkey patch default unittest.TestCase
  753. unittest.TestCase = TestCase
  754. class PsutilTestCase(TestCase):
  755. """Test class providing auto-cleanup wrappers on top of process
  756. test utilities.
  757. """
  758. def get_testfn(self, suffix="", dir=None):
  759. fname = get_testfn(suffix=suffix, dir=dir)
  760. self.addCleanup(safe_rmpath, fname)
  761. return fname
  762. def spawn_testproc(self, *args, **kwds):
  763. sproc = spawn_testproc(*args, **kwds)
  764. self.addCleanup(terminate, sproc)
  765. return sproc
  766. def spawn_children_pair(self):
  767. child1, child2 = spawn_children_pair()
  768. self.addCleanup(terminate, child2)
  769. self.addCleanup(terminate, child1) # executed first
  770. return (child1, child2)
  771. def spawn_zombie(self):
  772. parent, zombie = spawn_zombie()
  773. self.addCleanup(terminate, zombie)
  774. self.addCleanup(terminate, parent) # executed first
  775. return (parent, zombie)
  776. def pyrun(self, *args, **kwds):
  777. sproc, srcfile = pyrun(*args, **kwds)
  778. self.addCleanup(safe_rmpath, srcfile)
  779. self.addCleanup(terminate, sproc) # executed first
  780. return sproc
  781. def assertProcessGone(self, proc):
  782. self.assertRaises(psutil.NoSuchProcess, psutil.Process, proc.pid)
  783. if isinstance(proc, (psutil.Process, psutil.Popen)):
  784. assert not proc.is_running()
  785. self.assertRaises(psutil.NoSuchProcess, proc.status)
  786. proc.wait(timeout=0) # assert not raise TimeoutExpired
  787. assert not psutil.pid_exists(proc.pid), proc.pid
  788. self.assertNotIn(proc.pid, psutil.pids())
  789. @unittest.skipIf(PYPY, "unreliable on PYPY")
  790. class TestMemoryLeak(PsutilTestCase):
  791. """Test framework class for detecting function memory leaks,
  792. typically functions implemented in C which forgot to free() memory
  793. from the heap. It does so by checking whether the process memory
  794. usage increased before and after calling the function many times.
  795. Note that this is hard (probably impossible) to do reliably, due
  796. to how the OS handles memory, the GC and so on (memory can even
  797. decrease!). In order to avoid false positives, in case of failure
  798. (mem > 0) we retry the test for up to 5 times, increasing call
  799. repetitions each time. If the memory keeps increasing then it's a
  800. failure.
  801. If available (Linux, OSX, Windows), USS memory is used for comparison,
  802. since it's supposed to be more precise, see:
  803. https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python
  804. If not, RSS memory is used. mallinfo() on Linux and _heapwalk() on
  805. Windows may give even more precision, but at the moment are not
  806. implemented.
  807. PyPy appears to be completely unstable for this framework, probably
  808. because of its JIT, so tests on PYPY are skipped.
  809. Usage:
  810. class TestLeaks(psutil.tests.TestMemoryLeak):
  811. def test_fun(self):
  812. self.execute(some_function)
  813. """
  814. # Configurable class attrs.
  815. times = 200
  816. warmup_times = 10
  817. tolerance = 0 # memory
  818. retries = 10 if CI_TESTING else 5
  819. verbose = True
  820. _thisproc = psutil.Process()
  821. def _get_mem(self):
  822. # USS is the closest thing we have to "real" memory usage and it
  823. # should be less likely to produce false positives.
  824. mem = self._thisproc.memory_full_info()
  825. return getattr(mem, "uss", mem.rss)
  826. def _get_num_fds(self):
  827. if POSIX:
  828. return self._thisproc.num_fds()
  829. else:
  830. return self._thisproc.num_handles()
  831. def _log(self, msg):
  832. if self.verbose:
  833. print_color(msg, color="yellow", file=sys.stderr)
  834. def _check_fds(self, fun):
  835. """Makes sure num_fds() (POSIX) or num_handles() (Windows) does
  836. not increase after calling a function. Used to discover forgotten
  837. close(2) and CloseHandle syscalls.
  838. """
  839. before = self._get_num_fds()
  840. self.call(fun)
  841. after = self._get_num_fds()
  842. diff = after - before
  843. if diff < 0:
  844. raise self.fail("negative diff %r (gc probably collected a "
  845. "resource from a previous test)" % diff)
  846. if diff > 0:
  847. type_ = "fd" if POSIX else "handle"
  848. if diff > 1:
  849. type_ += "s"
  850. msg = "%s unclosed %s after calling %r" % (diff, type_, fun)
  851. raise self.fail(msg)
  852. def _call_ntimes(self, fun, times):
  853. """Get 2 distinct memory samples, before and after having
  854. called fun repeadetly, and return the memory difference.
  855. """
  856. gc.collect(generation=1)
  857. mem1 = self._get_mem()
  858. for x in range(times):
  859. ret = self.call(fun)
  860. del x, ret
  861. gc.collect(generation=1)
  862. mem2 = self._get_mem()
  863. self.assertEqual(gc.garbage, [])
  864. diff = mem2 - mem1 # can also be negative
  865. return diff
  866. def _check_mem(self, fun, times, warmup_times, retries, tolerance):
  867. messages = []
  868. prev_mem = 0
  869. increase = times
  870. for idx in range(1, retries + 1):
  871. mem = self._call_ntimes(fun, times)
  872. msg = "Run #%s: extra-mem=%s, per-call=%s, calls=%s" % (
  873. idx, bytes2human(mem), bytes2human(mem / times), times)
  874. messages.append(msg)
  875. success = mem <= tolerance or mem <= prev_mem
  876. if success:
  877. if idx > 1:
  878. self._log(msg)
  879. return
  880. else:
  881. if idx == 1:
  882. print() # NOQA
  883. self._log(msg)
  884. times += increase
  885. prev_mem = mem
  886. raise self.fail(". ".join(messages))
  887. # ---
  888. def call(self, fun):
  889. return fun()
  890. def execute(self, fun, times=None, warmup_times=None, retries=None,
  891. tolerance=None):
  892. """Test a callable."""
  893. times = times if times is not None else self.times
  894. warmup_times = warmup_times if warmup_times is not None \
  895. else self.warmup_times
  896. retries = retries if retries is not None else self.retries
  897. tolerance = tolerance if tolerance is not None else self.tolerance
  898. try:
  899. assert times >= 1, "times must be >= 1"
  900. assert warmup_times >= 0, "warmup_times must be >= 0"
  901. assert retries >= 0, "retries must be >= 0"
  902. assert tolerance >= 0, "tolerance must be >= 0"
  903. except AssertionError as err:
  904. raise ValueError(str(err))
  905. self._call_ntimes(fun, warmup_times) # warm up
  906. self._check_fds(fun)
  907. self._check_mem(fun, times=times, warmup_times=warmup_times,
  908. retries=retries, tolerance=tolerance)
  909. def execute_w_exc(self, exc, fun, **kwargs):
  910. """Convenience method to test a callable while making sure it
  911. raises an exception on every call.
  912. """
  913. def call():
  914. self.assertRaises(exc, fun)
  915. self.execute(call, **kwargs)
  916. def print_sysinfo():
  917. import collections
  918. import datetime
  919. import getpass
  920. import platform
  921. info = collections.OrderedDict()
  922. info['OS'] = platform.system()
  923. if psutil.OSX:
  924. info['version'] = str(platform.mac_ver())
  925. elif psutil.WINDOWS:
  926. info['version'] = ' '.join(map(str, platform.win32_ver()))
  927. if hasattr(platform, 'win32_edition'):
  928. info['edition'] = platform.win32_edition()
  929. else:
  930. info['version'] = platform.version()
  931. if psutil.POSIX:
  932. info['kernel'] = '.'.join(map(str, get_kernel_version()))
  933. info['arch'] = ', '.join(
  934. list(platform.architecture()) + [platform.machine()])
  935. info['hostname'] = platform.node()
  936. info['python'] = ', '.join([
  937. platform.python_implementation(),
  938. platform.python_version(),
  939. platform.python_compiler()])
  940. if psutil.POSIX:
  941. s = platform.libc_ver()[1]
  942. if s:
  943. info['glibc'] = s
  944. info['fs-encoding'] = sys.getfilesystemencoding()
  945. info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
  946. info['user'] = getpass.getuser()
  947. info['pid'] = os.getpid()
  948. print("=" * 70) # NOQA
  949. for k, v in info.items():
  950. print("%-14s %s" % (k + ':', v)) # NOQA
  951. print("=" * 70) # NOQA
  952. def _get_eligible_cpu():
  953. p = psutil.Process()
  954. if hasattr(p, "cpu_num"):
  955. return p.cpu_num()
  956. elif hasattr(p, "cpu_affinity"):
  957. return p.cpu_affinity()[0]
  958. return 0
  959. class process_namespace:
  960. """A container that lists all Process class method names + some
  961. reasonable parameters to be called with. Utility methods (parent(),
  962. children(), ...) are excluded.
  963. >>> ns = process_namespace(psutil.Process())
  964. >>> for fun, name in ns.iter(ns.getters):
  965. ... fun()
  966. """
  967. utils = [
  968. ('cpu_percent', (), {}),
  969. ('memory_percent', (), {}),
  970. ]
  971. ignored = [
  972. ('as_dict', (), {}),
  973. ('children', (), {'recursive': True}),
  974. ('is_running', (), {}),
  975. ('memory_info_ex', (), {}),
  976. ('oneshot', (), {}),
  977. ('parent', (), {}),
  978. ('parents', (), {}),
  979. ('pid', (), {}),
  980. ('wait', (0, ), {}),
  981. ]
  982. getters = [
  983. ('cmdline', (), {}),
  984. ('connections', (), {'kind': 'all'}),
  985. ('cpu_times', (), {}),
  986. ('create_time', (), {}),
  987. ('cwd', (), {}),
  988. ('exe', (), {}),
  989. ('memory_full_info', (), {}),
  990. ('memory_info', (), {}),
  991. ('name', (), {}),
  992. ('nice', (), {}),
  993. ('num_ctx_switches', (), {}),
  994. ('num_threads', (), {}),
  995. ('open_files', (), {}),
  996. ('ppid', (), {}),
  997. ('status', (), {}),
  998. ('threads', (), {}),
  999. ('username', (), {}),
  1000. ]
  1001. if POSIX:
  1002. getters += [('uids', (), {})]
  1003. getters += [('gids', (), {})]
  1004. getters += [('terminal', (), {})]
  1005. getters += [('num_fds', (), {})]
  1006. if HAS_PROC_IO_COUNTERS:
  1007. getters += [('io_counters', (), {})]
  1008. if HAS_IONICE:
  1009. getters += [('ionice', (), {})]
  1010. if HAS_RLIMIT:
  1011. getters += [('rlimit', (psutil.RLIMIT_NOFILE, ), {})]
  1012. if HAS_CPU_AFFINITY:
  1013. getters += [('cpu_affinity', (), {})]
  1014. if HAS_PROC_CPU_NUM:
  1015. getters += [('cpu_num', (), {})]
  1016. if HAS_ENVIRON:
  1017. getters += [('environ', (), {})]
  1018. if WINDOWS:
  1019. getters += [('num_handles', (), {})]
  1020. if HAS_MEMORY_MAPS:
  1021. getters += [('memory_maps', (), {'grouped': False})]
  1022. setters = []
  1023. if POSIX:
  1024. setters += [('nice', (0, ), {})]
  1025. else:
  1026. setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS, ), {})]
  1027. if HAS_RLIMIT:
  1028. setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})]
  1029. if HAS_IONICE:
  1030. if LINUX:
  1031. setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})]
  1032. else:
  1033. setters += [('ionice', (psutil.IOPRIO_NORMAL, ), {})]
  1034. if HAS_CPU_AFFINITY:
  1035. setters += [('cpu_affinity', ([_get_eligible_cpu()], ), {})]
  1036. killers = [
  1037. ('send_signal', (signal.SIGTERM, ), {}),
  1038. ('suspend', (), {}),
  1039. ('resume', (), {}),
  1040. ('terminate', (), {}),
  1041. ('kill', (), {}),
  1042. ]
  1043. if WINDOWS:
  1044. killers += [('send_signal', (signal.CTRL_C_EVENT, ), {})]
  1045. killers += [('send_signal', (signal.CTRL_BREAK_EVENT, ), {})]
  1046. all = utils + getters + setters + killers
  1047. def __init__(self, proc):
  1048. self._proc = proc
  1049. def iter(self, ls, clear_cache=True):
  1050. """Given a list of tuples yields a set of (fun, fun_name) tuples
  1051. in random order.
  1052. """
  1053. ls = list(ls)
  1054. random.shuffle(ls)
  1055. for fun_name, args, kwds in ls:
  1056. if clear_cache:
  1057. self.clear_cache()
  1058. fun = getattr(self._proc, fun_name)
  1059. fun = functools.partial(fun, *args, **kwds)
  1060. yield (fun, fun_name)
  1061. def clear_cache(self):
  1062. """Clear the cache of a Process instance."""
  1063. self._proc._init(self._proc.pid, _ignore_nsp=True)
  1064. @classmethod
  1065. def test_class_coverage(cls, test_class, ls):
  1066. """Given a TestCase instance and a list of tuples checks that
  1067. the class defines the required test method names.
  1068. """
  1069. for fun_name, _, _ in ls:
  1070. meth_name = 'test_' + fun_name
  1071. if not hasattr(test_class, meth_name):
  1072. msg = "%r class should define a '%s' method" % (
  1073. test_class.__class__.__name__, meth_name)
  1074. raise AttributeError(msg)
  1075. @classmethod
  1076. def test(cls):
  1077. this = set([x[0] for x in cls.all])
  1078. ignored = set([x[0] for x in cls.ignored])
  1079. klass = set([x for x in dir(psutil.Process) if x[0] != '_'])
  1080. leftout = (this | ignored) ^ klass
  1081. if leftout:
  1082. raise ValueError("uncovered Process class names: %r" % leftout)
  1083. class system_namespace:
  1084. """A container that lists all the module-level, system-related APIs.
  1085. Utilities such as cpu_percent() are excluded. Usage:
  1086. >>> ns = system_namespace
  1087. >>> for fun, name in ns.iter(ns.getters):
  1088. ... fun()
  1089. """
  1090. getters = [
  1091. ('boot_time', (), {}),
  1092. ('cpu_count', (), {'logical': False}),
  1093. ('cpu_count', (), {'logical': True}),
  1094. ('cpu_stats', (), {}),
  1095. ('cpu_times', (), {'percpu': False}),
  1096. ('cpu_times', (), {'percpu': True}),
  1097. ('disk_io_counters', (), {'perdisk': True}),
  1098. ('disk_partitions', (), {'all': True}),
  1099. ('disk_usage', (os.getcwd(), ), {}),
  1100. ('net_connections', (), {'kind': 'all'}),
  1101. ('net_if_addrs', (), {}),
  1102. ('net_if_stats', (), {}),
  1103. ('net_io_counters', (), {'pernic': True}),
  1104. ('pid_exists', (os.getpid(), ), {}),
  1105. ('pids', (), {}),
  1106. ('swap_memory', (), {}),
  1107. ('users', (), {}),
  1108. ('virtual_memory', (), {}),
  1109. ]
  1110. if HAS_CPU_FREQ:
  1111. getters += [('cpu_freq', (), {'percpu': True})]
  1112. if HAS_GETLOADAVG:
  1113. getters += [('getloadavg', (), {})]
  1114. if HAS_SENSORS_TEMPERATURES:
  1115. getters += [('sensors_temperatures', (), {})]
  1116. if HAS_SENSORS_FANS:
  1117. getters += [('sensors_fans', (), {})]
  1118. if HAS_SENSORS_BATTERY:
  1119. getters += [('sensors_battery', (), {})]
  1120. if WINDOWS:
  1121. getters += [('win_service_iter', (), {})]
  1122. getters += [('win_service_get', ('alg', ), {})]
  1123. ignored = [
  1124. ('process_iter', (), {}),
  1125. ('wait_procs', ([psutil.Process()], ), {}),
  1126. ('cpu_percent', (), {}),
  1127. ('cpu_times_percent', (), {}),
  1128. ]
  1129. all = getters
  1130. @staticmethod
  1131. def iter(ls):
  1132. """Given a list of tuples yields a set of (fun, fun_name) tuples
  1133. in random order.
  1134. """
  1135. ls = list(ls)
  1136. random.shuffle(ls)
  1137. for fun_name, args, kwds in ls:
  1138. fun = getattr(psutil, fun_name)
  1139. fun = functools.partial(fun, *args, **kwds)
  1140. yield (fun, fun_name)
  1141. test_class_coverage = process_namespace.test_class_coverage
  1142. def serialrun(klass):
  1143. """A decorator to mark a TestCase class. When running parallel tests,
  1144. class' unit tests will be run serially (1 process).
  1145. """
  1146. # assert issubclass(klass, unittest.TestCase), klass
  1147. assert inspect.isclass(klass), klass
  1148. klass._serialrun = True
  1149. return klass
  1150. def retry_on_failure(retries=NO_RETRIES):
  1151. """Decorator which runs a test function and retries N times before
  1152. actually failing.
  1153. """
  1154. def logfun(exc):
  1155. print("%r, retrying" % exc, file=sys.stderr) # NOQA
  1156. return retry(exception=AssertionError, timeout=None, retries=retries,
  1157. logfun=logfun)
  1158. def skip_on_access_denied(only_if=None):
  1159. """Decorator to Ignore AccessDenied exceptions."""
  1160. def decorator(fun):
  1161. @functools.wraps(fun)
  1162. def wrapper(*args, **kwargs):
  1163. try:
  1164. return fun(*args, **kwargs)
  1165. except psutil.AccessDenied:
  1166. if only_if is not None:
  1167. if not only_if:
  1168. raise
  1169. raise unittest.SkipTest("raises AccessDenied")
  1170. return wrapper
  1171. return decorator
  1172. def skip_on_not_implemented(only_if=None):
  1173. """Decorator to Ignore NotImplementedError exceptions."""
  1174. def decorator(fun):
  1175. @functools.wraps(fun)
  1176. def wrapper(*args, **kwargs):
  1177. try:
  1178. return fun(*args, **kwargs)
  1179. except NotImplementedError:
  1180. if only_if is not None:
  1181. if not only_if:
  1182. raise
  1183. msg = "%r was skipped because it raised NotImplementedError" \
  1184. % fun.__name__
  1185. raise unittest.SkipTest(msg)
  1186. return wrapper
  1187. return decorator
  1188. # ===================================================================
  1189. # --- network
  1190. # ===================================================================
  1191. def get_free_port(host='127.0.0.1'):
  1192. """Return an unused TCP port."""
  1193. with contextlib.closing(socket.socket()) as sock:
  1194. sock.bind((host, 0))
  1195. return sock.getsockname()[1]
  1196. def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None):
  1197. """Binds a generic socket."""
  1198. if addr is None and family in (AF_INET, AF_INET6):
  1199. addr = ("", 0)
  1200. sock = socket.socket(family, type)
  1201. try:
  1202. if os.name not in ('nt', 'cygwin'):
  1203. sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
  1204. sock.bind(addr)
  1205. if type == socket.SOCK_STREAM:
  1206. sock.listen(5)
  1207. return sock
  1208. except Exception:
  1209. sock.close()
  1210. raise
  1211. def bind_unix_socket(name, type=socket.SOCK_STREAM):
  1212. """Bind a UNIX socket."""
  1213. assert psutil.POSIX
  1214. assert not os.path.exists(name), name
  1215. sock = socket.socket(socket.AF_UNIX, type)
  1216. try:
  1217. sock.bind(name)
  1218. if type == socket.SOCK_STREAM:
  1219. sock.listen(5)
  1220. except Exception:
  1221. sock.close()
  1222. raise
  1223. return sock
  1224. def tcp_socketpair(family, addr=("", 0)):
  1225. """Build a pair of TCP sockets connected to each other.
  1226. Return a (server, client) tuple.
  1227. """
  1228. with contextlib.closing(socket.socket(family, SOCK_STREAM)) as ll:
  1229. ll.bind(addr)
  1230. ll.listen(5)
  1231. addr = ll.getsockname()
  1232. c = socket.socket(family, SOCK_STREAM)
  1233. try:
  1234. c.connect(addr)
  1235. caddr = c.getsockname()
  1236. while True:
  1237. a, addr = ll.accept()
  1238. # check that we've got the correct client
  1239. if addr == caddr:
  1240. return (a, c)
  1241. a.close()
  1242. except OSError:
  1243. c.close()
  1244. raise
  1245. def unix_socketpair(name):
  1246. """Build a pair of UNIX sockets connected to each other through
  1247. the same UNIX file name.
  1248. Return a (server, client) tuple.
  1249. """
  1250. assert psutil.POSIX
  1251. server = client = None
  1252. try:
  1253. server = bind_unix_socket(name, type=socket.SOCK_STREAM)
  1254. server.setblocking(0)
  1255. client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
  1256. client.setblocking(0)
  1257. client.connect(name)
  1258. # new = server.accept()
  1259. except Exception:
  1260. if server is not None:
  1261. server.close()
  1262. if client is not None:
  1263. client.close()
  1264. raise
  1265. return (server, client)
  1266. @contextlib.contextmanager
  1267. def create_sockets():
  1268. """Open as many socket families / types as possible."""
  1269. socks = []
  1270. fname1 = fname2 = None
  1271. try:
  1272. socks.append(bind_socket(socket.AF_INET, socket.SOCK_STREAM))
  1273. socks.append(bind_socket(socket.AF_INET, socket.SOCK_DGRAM))
  1274. if supports_ipv6():
  1275. socks.append(bind_socket(socket.AF_INET6, socket.SOCK_STREAM))
  1276. socks.append(bind_socket(socket.AF_INET6, socket.SOCK_DGRAM))
  1277. if POSIX and HAS_CONNECTIONS_UNIX:
  1278. fname1 = get_testfn()
  1279. fname2 = get_testfn()
  1280. s1, s2 = unix_socketpair(fname1)
  1281. s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM)
  1282. for s in (s1, s2, s3):
  1283. socks.append(s)
  1284. yield socks
  1285. finally:
  1286. for s in socks:
  1287. s.close()
  1288. for fname in (fname1, fname2):
  1289. if fname is not None:
  1290. safe_rmpath(fname)
  1291. def check_net_address(addr, family):
  1292. """Check a net address validity. Supported families are IPv4,
  1293. IPv6 and MAC addresses.
  1294. """
  1295. import ipaddress # python >= 3.3 / requires "pip install ipaddress"
  1296. if enum and PY3 and not PYPY:
  1297. assert isinstance(family, enum.IntEnum), family
  1298. if family == socket.AF_INET:
  1299. octs = [int(x) for x in addr.split('.')]
  1300. assert len(octs) == 4, addr
  1301. for num in octs:
  1302. assert 0 <= num <= 255, addr
  1303. if not PY3:
  1304. addr = unicode(addr)
  1305. ipaddress.IPv4Address(addr)
  1306. elif family == socket.AF_INET6:
  1307. assert isinstance(addr, str), addr
  1308. if not PY3:
  1309. addr = unicode(addr)
  1310. ipaddress.IPv6Address(addr)
  1311. elif family == psutil.AF_LINK:
  1312. assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr
  1313. else:
  1314. raise ValueError("unknown family %r", family)
  1315. # ===================================================================
  1316. # --- compatibility
  1317. # ===================================================================
  1318. def reload_module(module):
  1319. """Backport of importlib.reload of Python 3.3+."""
  1320. try:
  1321. import importlib
  1322. if not hasattr(importlib, 'reload'): # python <=3.3
  1323. raise ImportError
  1324. except ImportError:
  1325. import imp
  1326. return imp.reload(module)
  1327. else:
  1328. return importlib.reload(module)
  1329. def import_module_by_path(path):
  1330. name = os.path.splitext(os.path.basename(path))[0]
  1331. if sys.version_info[0] == 2:
  1332. import imp
  1333. return imp.load_source(name, path)
  1334. elif sys.version_info[:2] <= (3, 4):
  1335. from importlib.machinery import SourceFileLoader
  1336. return SourceFileLoader(name, path).load_module()
  1337. else:
  1338. import importlib.util
  1339. spec = importlib.util.spec_from_file_location(name, path)
  1340. mod = importlib.util.module_from_spec(spec)
  1341. spec.loader.exec_module(mod)
  1342. return mod
  1343. # ===================================================================
  1344. # --- others
  1345. # ===================================================================
  1346. def warn(msg):
  1347. """Raise a warning msg."""
  1348. warnings.warn(msg, UserWarning)
  1349. def is_namedtuple(x):
  1350. """Check if object is an instance of namedtuple."""
  1351. t = type(x)
  1352. b = t.__bases__
  1353. if len(b) != 1 or b[0] != tuple:
  1354. return False
  1355. f = getattr(t, '_fields', None)
  1356. if not isinstance(f, tuple):
  1357. return False
  1358. return all(type(n) == str for n in f)
  1359. if POSIX:
  1360. @contextlib.contextmanager
  1361. def copyload_shared_lib(suffix=""):
  1362. """Ctx manager which picks up a random shared CO lib used
  1363. by this process, copies it in another location and loads it
  1364. in memory via ctypes. Return the new absolutized path.
  1365. """
  1366. exe = 'pypy' if PYPY else 'python'
  1367. ext = ".so"
  1368. dst = get_testfn(suffix=suffix + ext)
  1369. libs = [x.path for x in psutil.Process().memory_maps() if
  1370. os.path.splitext(x.path)[1] == ext and
  1371. exe in x.path.lower()]
  1372. src = random.choice(libs)
  1373. shutil.copyfile(src, dst)
  1374. try:
  1375. ctypes.CDLL(dst)
  1376. yield dst
  1377. finally:
  1378. safe_rmpath(dst)
  1379. else:
  1380. @contextlib.contextmanager
  1381. def copyload_shared_lib(suffix=""):
  1382. """Ctx manager which picks up a random shared DLL lib used
  1383. by this process, copies it in another location and loads it
  1384. in memory via ctypes.
  1385. Return the new absolutized, normcased path.
  1386. """
  1387. from ctypes import wintypes
  1388. from ctypes import WinError
  1389. ext = ".dll"
  1390. dst = get_testfn(suffix=suffix + ext)
  1391. libs = [x.path for x in psutil.Process().memory_maps() if
  1392. x.path.lower().endswith(ext) and
  1393. 'python' in os.path.basename(x.path).lower() and
  1394. 'wow64' not in x.path.lower()]
  1395. if PYPY and not libs:
  1396. libs = [x.path for x in psutil.Process().memory_maps() if
  1397. 'pypy' in os.path.basename(x.path).lower()]
  1398. src = random.choice(libs)
  1399. shutil.copyfile(src, dst)
  1400. cfile = None
  1401. try:
  1402. cfile = ctypes.WinDLL(dst)
  1403. yield dst
  1404. finally:
  1405. # Work around OverflowError:
  1406. # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/
  1407. # job/o53330pbnri9bcw7
  1408. # - http://bugs.python.org/issue30286
  1409. # - http://stackoverflow.com/questions/23522055
  1410. if cfile is not None:
  1411. FreeLibrary = ctypes.windll.kernel32.FreeLibrary
  1412. FreeLibrary.argtypes = [wintypes.HMODULE]
  1413. ret = FreeLibrary(cfile._handle)
  1414. if ret == 0:
  1415. WinError()
  1416. safe_rmpath(dst)
  1417. # ===================================================================
  1418. # --- Exit funs (first is executed last)
  1419. # ===================================================================
  1420. # this is executed first
  1421. @atexit.register
  1422. def cleanup_test_procs():
  1423. reap_children(recursive=True)
  1424. # atexit module does not execute exit functions in case of SIGTERM, which
  1425. # gets sent to test subprocesses, which is a problem if they import this
  1426. # module. With this it will. See:
  1427. # https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python
  1428. if POSIX:
  1429. signal.signal(signal.SIGTERM, lambda sig, frame: sys.exit(sig))