test_process.py 83 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634
  1. # Copyright (c) Twisted Matrix Laboratories.
  2. # See LICENSE for details.
  3. """
  4. Test running processes.
  5. @var CONCURRENT_PROCESS_TEST_COUNT: The number of concurrent processes to use
  6. to stress-test the spawnProcess API. This value is tuned to a number of
  7. processes which has been determined to stay below various
  8. experimentally-determined limitations of our supported platforms.
  9. Particularly, Windows XP seems to have some undocumented limitations which
  10. cause spurious failures if this value is pushed too high. U{Please see
  11. this ticket for a discussion of how we arrived at its current value.
  12. <http://twistedmatrix.com/trac/ticket/3404>}
  13. @var properEnv: A copy of L{os.environ} which has L{bytes} keys/values on POSIX
  14. platforms and native L{str} keys/values on Windows.
  15. """
  16. from __future__ import division, absolute_import, print_function
  17. import gzip
  18. import os
  19. import sys
  20. import signal
  21. import errno
  22. import gc
  23. import stat
  24. import operator
  25. try:
  26. import fcntl
  27. except ImportError:
  28. fcntl = process = None
  29. else:
  30. from twisted.internet import process
  31. from zope.interface.verify import verifyObject
  32. from io import BytesIO
  33. from twisted.python.log import msg
  34. from twisted.internet import reactor, protocol, error, interfaces, defer
  35. from twisted.trial import unittest
  36. from twisted.python import runtime, procutils
  37. from twisted.python.compat import _PY3, networkString, range, bytesEnviron
  38. from twisted.python.filepath import FilePath
  39. # Get the current Python executable as a bytestring.
  40. pyExe = FilePath(sys.executable)._asBytesPath()
  41. CONCURRENT_PROCESS_TEST_COUNT = 25
  42. if not runtime.platform.isWindows():
  43. properEnv = bytesEnviron()
  44. properEnv[b"PYTHONPATH"] = os.pathsep.join(sys.path).encode(
  45. sys.getfilesystemencoding())
  46. else:
  47. properEnv = dict(os.environ)
  48. properEnv["PYTHONPATH"] = os.pathsep.join(sys.path)
  49. class StubProcessProtocol(protocol.ProcessProtocol):
  50. """
  51. ProcessProtocol counter-implementation: all methods on this class raise an
  52. exception, so instances of this may be used to verify that only certain
  53. methods are called.
  54. """
  55. def outReceived(self, data):
  56. raise NotImplementedError()
  57. def errReceived(self, data):
  58. raise NotImplementedError()
  59. def inConnectionLost(self):
  60. raise NotImplementedError()
  61. def outConnectionLost(self):
  62. raise NotImplementedError()
  63. def errConnectionLost(self):
  64. raise NotImplementedError()
  65. class ProcessProtocolTests(unittest.TestCase):
  66. """
  67. Tests for behavior provided by the process protocol base class,
  68. L{protocol.ProcessProtocol}.
  69. """
  70. def test_interface(self):
  71. """
  72. L{ProcessProtocol} implements L{IProcessProtocol}.
  73. """
  74. verifyObject(interfaces.IProcessProtocol, protocol.ProcessProtocol())
  75. def test_outReceived(self):
  76. """
  77. Verify that when stdout is delivered to
  78. L{ProcessProtocol.childDataReceived}, it is forwarded to
  79. L{ProcessProtocol.outReceived}.
  80. """
  81. received = []
  82. class OutProtocol(StubProcessProtocol):
  83. def outReceived(self, data):
  84. received.append(data)
  85. bytesToSend = b"bytes"
  86. p = OutProtocol()
  87. p.childDataReceived(1, bytesToSend)
  88. self.assertEqual(received, [bytesToSend])
  89. def test_errReceived(self):
  90. """
  91. Similar to L{test_outReceived}, but for stderr.
  92. """
  93. received = []
  94. class ErrProtocol(StubProcessProtocol):
  95. def errReceived(self, data):
  96. received.append(data)
  97. bytesToSend = b"bytes"
  98. p = ErrProtocol()
  99. p.childDataReceived(2, bytesToSend)
  100. self.assertEqual(received, [bytesToSend])
  101. def test_inConnectionLost(self):
  102. """
  103. Verify that when stdin close notification is delivered to
  104. L{ProcessProtocol.childConnectionLost}, it is forwarded to
  105. L{ProcessProtocol.inConnectionLost}.
  106. """
  107. lost = []
  108. class InLostProtocol(StubProcessProtocol):
  109. def inConnectionLost(self):
  110. lost.append(None)
  111. p = InLostProtocol()
  112. p.childConnectionLost(0)
  113. self.assertEqual(lost, [None])
  114. def test_outConnectionLost(self):
  115. """
  116. Similar to L{test_inConnectionLost}, but for stdout.
  117. """
  118. lost = []
  119. class OutLostProtocol(StubProcessProtocol):
  120. def outConnectionLost(self):
  121. lost.append(None)
  122. p = OutLostProtocol()
  123. p.childConnectionLost(1)
  124. self.assertEqual(lost, [None])
  125. def test_errConnectionLost(self):
  126. """
  127. Similar to L{test_inConnectionLost}, but for stderr.
  128. """
  129. lost = []
  130. class ErrLostProtocol(StubProcessProtocol):
  131. def errConnectionLost(self):
  132. lost.append(None)
  133. p = ErrLostProtocol()
  134. p.childConnectionLost(2)
  135. self.assertEqual(lost, [None])
  136. class TrivialProcessProtocol(protocol.ProcessProtocol):
  137. """
  138. Simple process protocol for tests purpose.
  139. @ivar outData: data received from stdin
  140. @ivar errData: data received from stderr
  141. """
  142. def __init__(self, d):
  143. """
  144. Create the deferred that will be fired at the end, and initialize
  145. data structures.
  146. """
  147. self.deferred = d
  148. self.outData = []
  149. self.errData = []
  150. def processEnded(self, reason):
  151. self.reason = reason
  152. self.deferred.callback(None)
  153. def outReceived(self, data):
  154. self.outData.append(data)
  155. def errReceived(self, data):
  156. self.errData.append(data)
  157. class TestProcessProtocol(protocol.ProcessProtocol):
  158. def connectionMade(self):
  159. self.stages = [1]
  160. self.data = b''
  161. self.err = b''
  162. self.transport.write(b"abcd")
  163. def childDataReceived(self, childFD, data):
  164. """
  165. Override and disable the dispatch provided by the base class to ensure
  166. that it is really this method which is being called, and the transport
  167. is not going directly to L{outReceived} or L{errReceived}.
  168. """
  169. if childFD == 1:
  170. self.data += data
  171. elif childFD == 2:
  172. self.err += data
  173. def childConnectionLost(self, childFD):
  174. """
  175. Similarly to L{childDataReceived}, disable the automatic dispatch
  176. provided by the base implementation to verify that the transport is
  177. calling this method directly.
  178. """
  179. if childFD == 1:
  180. self.stages.append(2)
  181. if self.data != b"abcd":
  182. raise RuntimeError(
  183. "Data was %r instead of 'abcd'" % (self.data,))
  184. self.transport.write(b"1234")
  185. elif childFD == 2:
  186. self.stages.append(3)
  187. if self.err != b"1234":
  188. raise RuntimeError(
  189. "Err was %r instead of '1234'" % (self.err,))
  190. self.transport.write(b"abcd")
  191. self.stages.append(4)
  192. elif childFD == 0:
  193. self.stages.append(5)
  194. def processEnded(self, reason):
  195. self.reason = reason
  196. self.deferred.callback(None)
  197. class EchoProtocol(protocol.ProcessProtocol):
  198. s = b"1234567" * 1001
  199. n = 10
  200. finished = 0
  201. failure = None
  202. def __init__(self, onEnded):
  203. self.onEnded = onEnded
  204. self.count = 0
  205. def connectionMade(self):
  206. assert self.n > 2
  207. for i in range(self.n - 2):
  208. self.transport.write(self.s)
  209. # test writeSequence
  210. self.transport.writeSequence([self.s, self.s])
  211. self.buffer = self.s * self.n
  212. def outReceived(self, data):
  213. if self.buffer[self.count:self.count+len(data)] != data:
  214. self.failure = ("wrong bytes received", data, self.count)
  215. self.transport.closeStdin()
  216. else:
  217. self.count += len(data)
  218. if self.count == len(self.buffer):
  219. self.transport.closeStdin()
  220. def processEnded(self, reason):
  221. self.finished = 1
  222. if not reason.check(error.ProcessDone):
  223. self.failure = "process didn't terminate normally: " + str(reason)
  224. self.onEnded.callback(self)
  225. class SignalProtocol(protocol.ProcessProtocol):
  226. """
  227. A process protocol that sends a signal when data is first received.
  228. @ivar deferred: deferred firing on C{processEnded}.
  229. @type deferred: L{defer.Deferred}
  230. @ivar signal: the signal to send to the process.
  231. @type signal: C{str}
  232. @ivar signaled: A flag tracking whether the signal has been sent to the
  233. child or not yet. C{False} until it is sent, then C{True}.
  234. @type signaled: C{bool}
  235. """
  236. def __init__(self, deferred, sig):
  237. self.deferred = deferred
  238. self.signal = sig
  239. self.signaled = False
  240. def outReceived(self, data):
  241. """
  242. Handle the first output from the child process (which indicates it
  243. is set up and ready to receive the signal) by sending the signal to
  244. it. Also log all output to help with debugging.
  245. """
  246. msg("Received %r from child stdout" % (data,))
  247. if not self.signaled:
  248. self.signaled = True
  249. self.transport.signalProcess(self.signal)
  250. def errReceived(self, data):
  251. """
  252. Log all data received from the child's stderr to help with
  253. debugging.
  254. """
  255. msg("Received %r from child stderr" % (data,))
  256. def processEnded(self, reason):
  257. """
  258. Callback C{self.deferred} with L{None} if C{reason} is a
  259. L{error.ProcessTerminated} failure with C{exitCode} set to L{None},
  260. C{signal} set to C{self.signal}, and C{status} holding the status code
  261. of the exited process. Otherwise, errback with a C{ValueError}
  262. describing the problem.
  263. """
  264. msg("Child exited: %r" % (reason.getTraceback(),))
  265. if not reason.check(error.ProcessTerminated):
  266. return self.deferred.errback(
  267. ValueError("wrong termination: %s" % (reason,)))
  268. v = reason.value
  269. if isinstance(self.signal, str):
  270. signalValue = getattr(signal, 'SIG' + self.signal)
  271. else:
  272. signalValue = self.signal
  273. if v.exitCode is not None:
  274. return self.deferred.errback(
  275. ValueError("SIG%s: exitCode is %s, not None" %
  276. (self.signal, v.exitCode)))
  277. if v.signal != signalValue:
  278. return self.deferred.errback(
  279. ValueError("SIG%s: .signal was %s, wanted %s" %
  280. (self.signal, v.signal, signalValue)))
  281. if os.WTERMSIG(v.status) != signalValue:
  282. return self.deferred.errback(
  283. ValueError('SIG%s: %s' % (self.signal, os.WTERMSIG(v.status))))
  284. self.deferred.callback(None)
  285. class TestManyProcessProtocol(TestProcessProtocol):
  286. def __init__(self):
  287. self.deferred = defer.Deferred()
  288. def processEnded(self, reason):
  289. self.reason = reason
  290. if reason.check(error.ProcessDone):
  291. self.deferred.callback(None)
  292. else:
  293. self.deferred.errback(reason)
  294. class UtilityProcessProtocol(protocol.ProcessProtocol):
  295. """
  296. Helper class for launching a Python process and getting a result from it.
  297. @ivar programName: The name of the program to run.
  298. """
  299. programName = None
  300. @classmethod
  301. def run(cls, reactor, argv, env):
  302. """
  303. Run a Python process connected to a new instance of this protocol
  304. class. Return the protocol instance.
  305. The Python process is given C{self.program} on the command line to
  306. execute, in addition to anything specified by C{argv}. C{env} is
  307. the complete environment.
  308. """
  309. self = cls()
  310. reactor.spawnProcess(
  311. self, pyExe, [pyExe, b"-u", b"-m", self.programName] + argv,
  312. env=env)
  313. return self
  314. def __init__(self):
  315. self.bytes = []
  316. self.requests = []
  317. def parseChunks(self, bytes):
  318. """
  319. Called with all bytes received on stdout when the process exits.
  320. """
  321. raise NotImplementedError()
  322. def getResult(self):
  323. """
  324. Return a Deferred which will fire with the result of L{parseChunks}
  325. when the child process exits.
  326. """
  327. d = defer.Deferred()
  328. self.requests.append(d)
  329. return d
  330. def _fireResultDeferreds(self, result):
  331. """
  332. Callback all Deferreds returned up until now by L{getResult}
  333. with the given result object.
  334. """
  335. requests = self.requests
  336. self.requests = None
  337. for d in requests:
  338. d.callback(result)
  339. def outReceived(self, bytes):
  340. """
  341. Accumulate output from the child process in a list.
  342. """
  343. self.bytes.append(bytes)
  344. def processEnded(self, reason):
  345. """
  346. Handle process termination by parsing all received output and firing
  347. any waiting Deferreds.
  348. """
  349. self._fireResultDeferreds(self.parseChunks(self.bytes))
  350. class GetArgumentVector(UtilityProcessProtocol):
  351. """
  352. Protocol which will read a serialized argv from a process and
  353. expose it to interested parties.
  354. """
  355. programName = b"twisted.test.process_getargv"
  356. def parseChunks(self, chunks):
  357. """
  358. Parse the output from the process to which this protocol was
  359. connected, which is a single unterminated line of \\0-separated
  360. strings giving the argv of that process. Return this as a list of
  361. str objects.
  362. """
  363. return b''.join(chunks).split(b'\0')
  364. class GetEnvironmentDictionary(UtilityProcessProtocol):
  365. """
  366. Protocol which will read a serialized environment dict from a process
  367. and expose it to interested parties.
  368. """
  369. programName = b"twisted.test.process_getenv"
  370. def parseChunks(self, chunks):
  371. """
  372. Parse the output from the process to which this protocol was
  373. connected, which is a single unterminated line of \\0-separated
  374. strings giving key value pairs of the environment from that process.
  375. Return this as a dictionary.
  376. """
  377. environString = b''.join(chunks)
  378. if not environString:
  379. return {}
  380. environ = iter(environString.split(b'\0'))
  381. d = {}
  382. while 1:
  383. try:
  384. k = next(environ)
  385. except StopIteration:
  386. break
  387. else:
  388. v = next(environ)
  389. d[k] = v
  390. return d
  391. class ProcessTests(unittest.TestCase):
  392. """
  393. Test running a process.
  394. """
  395. usePTY = False
  396. def test_stdio(self):
  397. """
  398. L{twisted.internet.stdio} test.
  399. """
  400. scriptPath = b"twisted.test.process_twisted"
  401. p = Accumulator()
  402. d = p.endedDeferred = defer.Deferred()
  403. reactor.spawnProcess(p, pyExe, [pyExe, b'-u', b"-m", scriptPath],
  404. env=properEnv,
  405. path=None, usePTY=self.usePTY)
  406. p.transport.write(b"hello, world")
  407. p.transport.write(b"abc")
  408. p.transport.write(b"123")
  409. p.transport.closeStdin()
  410. def processEnded(ign):
  411. self.assertEqual(p.outF.getvalue(), b"hello, worldabc123",
  412. "Output follows:\n"
  413. "%s\n"
  414. "Error message from process_twisted follows:\n"
  415. "%s\n" % (p.outF.getvalue(), p.errF.getvalue()))
  416. return d.addCallback(processEnded)
  417. def test_unsetPid(self):
  418. """
  419. Test if pid is None/non-None before/after process termination. This
  420. reuses process_echoer.py to get a process that blocks on stdin.
  421. """
  422. finished = defer.Deferred()
  423. p = TrivialProcessProtocol(finished)
  424. scriptPath = b"twisted.test.process_echoer"
  425. procTrans = reactor.spawnProcess(p, pyExe,
  426. [pyExe, b'-u', b"-m", scriptPath],
  427. env=properEnv)
  428. self.assertTrue(procTrans.pid)
  429. def afterProcessEnd(ignored):
  430. self.assertIsNone(procTrans.pid)
  431. p.transport.closeStdin()
  432. return finished.addCallback(afterProcessEnd)
  433. def test_process(self):
  434. """
  435. Test running a process: check its output, it exitCode, some property of
  436. signalProcess.
  437. """
  438. scriptPath = b"twisted.test.process_tester"
  439. d = defer.Deferred()
  440. p = TestProcessProtocol()
  441. p.deferred = d
  442. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
  443. env=properEnv)
  444. def check(ignored):
  445. self.assertEqual(p.stages, [1, 2, 3, 4, 5])
  446. f = p.reason
  447. f.trap(error.ProcessTerminated)
  448. self.assertEqual(f.value.exitCode, 23)
  449. # would .signal be available on non-posix?
  450. # self.assertIsNone(f.value.signal)
  451. self.assertRaises(
  452. error.ProcessExitedAlready, p.transport.signalProcess, 'INT')
  453. try:
  454. import process_tester, glob
  455. for f in glob.glob(process_tester.test_file_match):
  456. os.remove(f)
  457. except:
  458. pass
  459. d.addCallback(check)
  460. return d
  461. def test_manyProcesses(self):
  462. def _check(results, protocols):
  463. for p in protocols:
  464. self.assertEqual(p.stages, [1, 2, 3, 4, 5],
  465. "[%d] stages = %s" % (id(p.transport),
  466. str(p.stages)))
  467. # test status code
  468. f = p.reason
  469. f.trap(error.ProcessTerminated)
  470. self.assertEqual(f.value.exitCode, 23)
  471. scriptPath = b"twisted.test.process_tester"
  472. args = [pyExe, b'-u', b"-m", scriptPath]
  473. protocols = []
  474. deferreds = []
  475. for i in range(CONCURRENT_PROCESS_TEST_COUNT):
  476. p = TestManyProcessProtocol()
  477. protocols.append(p)
  478. reactor.spawnProcess(p, pyExe, args, env=properEnv)
  479. deferreds.append(p.deferred)
  480. deferredList = defer.DeferredList(deferreds, consumeErrors=True)
  481. deferredList.addCallback(_check, protocols)
  482. return deferredList
  483. def test_echo(self):
  484. """
  485. A spawning a subprocess which echoes its stdin to its stdout via
  486. L{IReactorProcess.spawnProcess} will result in that echoed output being
  487. delivered to outReceived.
  488. """
  489. finished = defer.Deferred()
  490. p = EchoProtocol(finished)
  491. scriptPath = b"twisted.test.process_echoer"
  492. reactor.spawnProcess(p, pyExe, [pyExe, b'-u', b"-m", scriptPath],
  493. env=properEnv)
  494. def asserts(ignored):
  495. self.assertFalse(p.failure, p.failure)
  496. self.assertTrue(hasattr(p, 'buffer'))
  497. self.assertEqual(len(p.buffer), len(p.s * p.n))
  498. def takedownProcess(err):
  499. p.transport.closeStdin()
  500. return err
  501. return finished.addCallback(asserts).addErrback(takedownProcess)
  502. def test_commandLine(self):
  503. args = [br'a\"b ', br'a\b ', br' a\\"b', br' a\\b', br'"foo bar" "',
  504. b'\tab', b'"\\', b'a"b', b"a'b"]
  505. scriptPath = b"twisted.test.process_cmdline"
  506. p = Accumulator()
  507. d = p.endedDeferred = defer.Deferred()
  508. reactor.spawnProcess(p, pyExe,
  509. [pyExe, b"-u", b"-m", scriptPath] + args,
  510. env=properEnv, path=None)
  511. def processEnded(ign):
  512. self.assertEqual(p.errF.getvalue(), b"")
  513. recvdArgs = p.outF.getvalue().splitlines()
  514. self.assertEqual(recvdArgs, args)
  515. return d.addCallback(processEnded)
  516. def test_wrongArguments(self):
  517. """
  518. Test invalid arguments to spawnProcess: arguments and environment
  519. must only contains string or unicode, and not null bytes.
  520. """
  521. p = protocol.ProcessProtocol()
  522. badEnvs = [
  523. {b"foo": 2},
  524. {b"foo": b"egg\0a"},
  525. {3: b"bar"},
  526. {b"bar\0foo": b"bar"}]
  527. badArgs = [
  528. [pyExe, 2],
  529. b"spam",
  530. [pyExe, b"foo\0bar"]]
  531. # Sanity check - this will fail for people who have mucked with
  532. # their site configuration in a stupid way, but there's nothing we
  533. # can do about that.
  534. badUnicode = u'\N{SNOWMAN}'
  535. try:
  536. badUnicode.encode(sys.getfilesystemencoding())
  537. except UnicodeEncodeError:
  538. # Okay, that unicode doesn't encode, put it in as a bad environment
  539. # key.
  540. badEnvs.append({badUnicode: 'value for bad unicode key'})
  541. badEnvs.append({'key for bad unicode value': badUnicode})
  542. badArgs.append([pyExe, badUnicode])
  543. else:
  544. # It _did_ encode. Most likely, Gtk2 is being used and the
  545. # default system encoding is UTF-8, which can encode anything.
  546. # In any case, if implicit unicode -> str conversion works for
  547. # that string, we can't test that TypeError gets raised instead,
  548. # so just leave it off.
  549. pass
  550. for env in badEnvs:
  551. self.assertRaises(
  552. TypeError,
  553. reactor.spawnProcess, p, pyExe, [pyExe, b"-c", b""], env=env)
  554. for args in badArgs:
  555. self.assertRaises(
  556. TypeError,
  557. reactor.spawnProcess, p, pyExe, args, env=None)
  558. class TwoProcessProtocol(protocol.ProcessProtocol):
  559. num = -1
  560. finished = 0
  561. def __init__(self):
  562. self.deferred = defer.Deferred()
  563. def outReceived(self, data):
  564. pass
  565. def processEnded(self, reason):
  566. self.finished = 1
  567. self.deferred.callback(None)
  568. class TestTwoProcessesBase:
  569. def setUp(self):
  570. self.processes = [None, None]
  571. self.pp = [None, None]
  572. self.done = 0
  573. self.verbose = 0
  574. def createProcesses(self, usePTY=0):
  575. scriptPath = b"twisted.test.process_reader"
  576. for num in (0,1):
  577. self.pp[num] = TwoProcessProtocol()
  578. self.pp[num].num = num
  579. p = reactor.spawnProcess(self.pp[num], pyExe,
  580. [pyExe, b"-u", b"-m", scriptPath],
  581. env=properEnv, usePTY=usePTY)
  582. self.processes[num] = p
  583. def close(self, num):
  584. if self.verbose: print("closing stdin [%d]" % num)
  585. p = self.processes[num]
  586. pp = self.pp[num]
  587. self.assertFalse(pp.finished, "Process finished too early")
  588. p.loseConnection()
  589. if self.verbose: print(self.pp[0].finished, self.pp[1].finished)
  590. def _onClose(self):
  591. return defer.gatherResults([ p.deferred for p in self.pp ])
  592. def test_close(self):
  593. if self.verbose: print("starting processes")
  594. self.createProcesses()
  595. reactor.callLater(1, self.close, 0)
  596. reactor.callLater(2, self.close, 1)
  597. return self._onClose()
  598. class TwoProcessesNonPosixTests(TestTwoProcessesBase, unittest.TestCase):
  599. pass
  600. class TwoProcessesPosixTests(TestTwoProcessesBase, unittest.TestCase):
  601. def tearDown(self):
  602. for pp, pr in zip(self.pp, self.processes):
  603. if not pp.finished:
  604. try:
  605. os.kill(pr.pid, signal.SIGTERM)
  606. except OSError:
  607. # If the test failed the process may already be dead
  608. # The error here is only noise
  609. pass
  610. return self._onClose()
  611. def kill(self, num):
  612. if self.verbose: print("kill [%d] with SIGTERM" % num)
  613. p = self.processes[num]
  614. pp = self.pp[num]
  615. self.assertFalse(pp.finished, "Process finished too early")
  616. os.kill(p.pid, signal.SIGTERM)
  617. if self.verbose: print(self.pp[0].finished, self.pp[1].finished)
  618. def test_kill(self):
  619. if self.verbose: print("starting processes")
  620. self.createProcesses(usePTY=0)
  621. reactor.callLater(1, self.kill, 0)
  622. reactor.callLater(2, self.kill, 1)
  623. return self._onClose()
  624. def test_closePty(self):
  625. if self.verbose: print("starting processes")
  626. self.createProcesses(usePTY=1)
  627. reactor.callLater(1, self.close, 0)
  628. reactor.callLater(2, self.close, 1)
  629. return self._onClose()
  630. def test_killPty(self):
  631. if self.verbose: print("starting processes")
  632. self.createProcesses(usePTY=1)
  633. reactor.callLater(1, self.kill, 0)
  634. reactor.callLater(2, self.kill, 1)
  635. return self._onClose()
  636. class FDChecker(protocol.ProcessProtocol):
  637. state = 0
  638. data = b""
  639. failed = None
  640. def __init__(self, d):
  641. self.deferred = d
  642. def fail(self, why):
  643. self.failed = why
  644. self.deferred.callback(None)
  645. def connectionMade(self):
  646. self.transport.writeToChild(0, b"abcd")
  647. self.state = 1
  648. def childDataReceived(self, childFD, data):
  649. if self.state == 1:
  650. if childFD != 1:
  651. self.fail("read '%s' on fd %d (not 1) during state 1" \
  652. % (childFD, data))
  653. return
  654. self.data += data
  655. #print "len", len(self.data)
  656. if len(self.data) == 6:
  657. if self.data != b"righto":
  658. self.fail("got '%s' on fd1, expected 'righto'" \
  659. % self.data)
  660. return
  661. self.data = b""
  662. self.state = 2
  663. #print "state2", self.state
  664. self.transport.writeToChild(3, b"efgh")
  665. return
  666. if self.state == 2:
  667. self.fail("read '%s' on fd %s during state 2" % (childFD, data))
  668. return
  669. if self.state == 3:
  670. if childFD != 1:
  671. self.fail("read '%s' on fd %s (not 1) during state 3" \
  672. % (childFD, data))
  673. return
  674. self.data += data
  675. if len(self.data) == 6:
  676. if self.data != b"closed":
  677. self.fail("got '%s' on fd1, expected 'closed'" \
  678. % self.data)
  679. return
  680. self.state = 4
  681. return
  682. if self.state == 4:
  683. self.fail("read '%s' on fd %s during state 4" % (childFD, data))
  684. return
  685. def childConnectionLost(self, childFD):
  686. if self.state == 1:
  687. self.fail("got connectionLost(%d) during state 1" % childFD)
  688. return
  689. if self.state == 2:
  690. if childFD != 4:
  691. self.fail("got connectionLost(%d) (not 4) during state 2" \
  692. % childFD)
  693. return
  694. self.state = 3
  695. self.transport.closeChildFD(5)
  696. return
  697. def processEnded(self, status):
  698. rc = status.value.exitCode
  699. if self.state != 4:
  700. self.fail("processEnded early, rc %d" % rc)
  701. return
  702. if status.value.signal != None:
  703. self.fail("processEnded with signal %s" % status.value.signal)
  704. return
  705. if rc != 0:
  706. self.fail("processEnded with rc %d" % rc)
  707. return
  708. self.deferred.callback(None)
  709. class FDTests(unittest.TestCase):
  710. def test_FD(self):
  711. scriptPath = b"twisted.test.process_fds"
  712. d = defer.Deferred()
  713. p = FDChecker(d)
  714. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
  715. env=properEnv,
  716. childFDs={0:"w", 1:"r", 2:2,
  717. 3:"w", 4:"r", 5:"w"})
  718. d.addCallback(lambda x : self.assertFalse(p.failed, p.failed))
  719. return d
  720. def test_linger(self):
  721. # See what happens when all the pipes close before the process
  722. # actually stops. This test *requires* SIGCHLD catching to work,
  723. # as there is no other way to find out the process is done.
  724. scriptPath = b"twisted.test.process_linger"
  725. p = Accumulator()
  726. d = p.endedDeferred = defer.Deferred()
  727. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
  728. env=properEnv,
  729. childFDs={1:"r", 2:2},
  730. )
  731. def processEnded(ign):
  732. self.assertEqual(p.outF.getvalue(),
  733. b"here is some text\ngoodbye\n")
  734. return d.addCallback(processEnded)
  735. class Accumulator(protocol.ProcessProtocol):
  736. """Accumulate data from a process."""
  737. closed = 0
  738. endedDeferred = None
  739. def connectionMade(self):
  740. self.outF = BytesIO()
  741. self.errF = BytesIO()
  742. def outReceived(self, d):
  743. self.outF.write(d)
  744. def errReceived(self, d):
  745. self.errF.write(d)
  746. def outConnectionLost(self):
  747. pass
  748. def errConnectionLost(self):
  749. pass
  750. def processEnded(self, reason):
  751. self.closed = 1
  752. if self.endedDeferred is not None:
  753. d, self.endedDeferred = self.endedDeferred, None
  754. d.callback(None)
  755. class PosixProcessBase(object):
  756. """
  757. Test running processes.
  758. """
  759. usePTY = False
  760. def getCommand(self, commandName):
  761. """
  762. Return the path of the shell command named C{commandName}, looking at
  763. common locations.
  764. """
  765. binLoc = FilePath('/bin').child(commandName)
  766. usrbinLoc = FilePath('/usr/bin').child(commandName)
  767. if binLoc.exists():
  768. return binLoc._asBytesPath()
  769. elif usrbinLoc.exists():
  770. return usrbinLoc._asBytesPath()
  771. else:
  772. raise RuntimeError(
  773. "%s not found in /bin or /usr/bin" % (commandName,))
  774. def test_normalTermination(self):
  775. cmd = self.getCommand('true')
  776. d = defer.Deferred()
  777. p = TrivialProcessProtocol(d)
  778. reactor.spawnProcess(p, cmd, [b'true'], env=None,
  779. usePTY=self.usePTY)
  780. def check(ignored):
  781. p.reason.trap(error.ProcessDone)
  782. self.assertEqual(p.reason.value.exitCode, 0)
  783. self.assertIsNone(p.reason.value.signal)
  784. d.addCallback(check)
  785. return d
  786. def test_abnormalTermination(self):
  787. """
  788. When a process terminates with a system exit code set to 1,
  789. C{processEnded} is called with a L{error.ProcessTerminated} error,
  790. the C{exitCode} attribute reflecting the system exit code.
  791. """
  792. d = defer.Deferred()
  793. p = TrivialProcessProtocol(d)
  794. reactor.spawnProcess(p, pyExe,
  795. [pyExe, b'-c', b'import sys; sys.exit(1)'],
  796. env=None, usePTY=self.usePTY)
  797. def check(ignored):
  798. p.reason.trap(error.ProcessTerminated)
  799. self.assertEqual(p.reason.value.exitCode, 1)
  800. self.assertIsNone(p.reason.value.signal)
  801. d.addCallback(check)
  802. return d
  803. def _testSignal(self, sig):
  804. scriptPath = b"twisted.test.process_signal"
  805. d = defer.Deferred()
  806. p = SignalProtocol(d, sig)
  807. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", "-m", scriptPath],
  808. env=properEnv, usePTY=self.usePTY)
  809. return d
  810. def test_signalHUP(self):
  811. """
  812. Sending the SIGHUP signal to a running process interrupts it, and
  813. C{processEnded} is called with a L{error.ProcessTerminated} instance
  814. with the C{exitCode} set to L{None} and the C{signal} attribute set to
  815. C{signal.SIGHUP}. C{os.WTERMSIG} can also be used on the C{status}
  816. attribute to extract the signal value.
  817. """
  818. return self._testSignal('HUP')
  819. def test_signalINT(self):
  820. """
  821. Sending the SIGINT signal to a running process interrupts it, and
  822. C{processEnded} is called with a L{error.ProcessTerminated} instance
  823. with the C{exitCode} set to L{None} and the C{signal} attribute set to
  824. C{signal.SIGINT}. C{os.WTERMSIG} can also be used on the C{status}
  825. attribute to extract the signal value.
  826. """
  827. return self._testSignal('INT')
  828. def test_signalKILL(self):
  829. """
  830. Sending the SIGKILL signal to a running process interrupts it, and
  831. C{processEnded} is called with a L{error.ProcessTerminated} instance
  832. with the C{exitCode} set to L{None} and the C{signal} attribute set to
  833. C{signal.SIGKILL}. C{os.WTERMSIG} can also be used on the C{status}
  834. attribute to extract the signal value.
  835. """
  836. return self._testSignal('KILL')
  837. def test_signalTERM(self):
  838. """
  839. Sending the SIGTERM signal to a running process interrupts it, and
  840. C{processEnded} is called with a L{error.ProcessTerminated} instance
  841. with the C{exitCode} set to L{None} and the C{signal} attribute set to
  842. C{signal.SIGTERM}. C{os.WTERMSIG} can also be used on the C{status}
  843. attribute to extract the signal value.
  844. """
  845. return self._testSignal('TERM')
  846. def test_childSignalHandling(self):
  847. """
  848. The disposition of signals which are ignored in the parent
  849. process is reset to the default behavior for the child
  850. process.
  851. """
  852. # Somewhat arbitrarily select SIGUSR1 here. It satisfies our
  853. # requirements that:
  854. # - The interpreter not fiddle around with the handler
  855. # behind our backs at startup time (this disqualifies
  856. # signals like SIGINT and SIGPIPE).
  857. # - The default behavior is to exit.
  858. #
  859. # This lets us send the signal to the child and then verify
  860. # that it exits with a status code indicating that it was
  861. # indeed the signal which caused it to exit.
  862. which = signal.SIGUSR1
  863. # Ignore the signal in the parent (and make sure we clean it
  864. # up).
  865. handler = signal.signal(which, signal.SIG_IGN)
  866. self.addCleanup(signal.signal, signal.SIGUSR1, handler)
  867. # Now do the test.
  868. return self._testSignal(signal.SIGUSR1)
  869. def test_executionError(self):
  870. """
  871. Raise an error during execvpe to check error management.
  872. """
  873. cmd = self.getCommand('false')
  874. d = defer.Deferred()
  875. p = TrivialProcessProtocol(d)
  876. def buggyexecvpe(command, args, environment):
  877. raise RuntimeError("Ouch")
  878. oldexecvpe = os.execvpe
  879. os.execvpe = buggyexecvpe
  880. try:
  881. reactor.spawnProcess(p, cmd, [b'false'], env=None,
  882. usePTY=self.usePTY)
  883. def check(ignored):
  884. errData = b"".join(p.errData + p.outData)
  885. self.assertIn(b"Upon execvpe", errData)
  886. self.assertIn(b"Ouch", errData)
  887. d.addCallback(check)
  888. finally:
  889. os.execvpe = oldexecvpe
  890. return d
  891. if runtime.platform.isMacOSX():
  892. test_executionError.skip = (
  893. "Test is flaky from a Darwin bug. See #8840.")
  894. def test_errorInProcessEnded(self):
  895. """
  896. The handler which reaps a process is removed when the process is
  897. reaped, even if the protocol's C{processEnded} method raises an
  898. exception.
  899. """
  900. connected = defer.Deferred()
  901. ended = defer.Deferred()
  902. # This script runs until we disconnect its transport.
  903. scriptPath = b"twisted.test.process_echoer"
  904. class ErrorInProcessEnded(protocol.ProcessProtocol):
  905. """
  906. A protocol that raises an error in C{processEnded}.
  907. """
  908. def makeConnection(self, transport):
  909. connected.callback(transport)
  910. def processEnded(self, reason):
  911. reactor.callLater(0, ended.callback, None)
  912. raise RuntimeError("Deliberate error")
  913. # Launch the process.
  914. reactor.spawnProcess(
  915. ErrorInProcessEnded(), pyExe,
  916. [pyExe, b"-u", b"-m", scriptPath],
  917. env=properEnv, path=None)
  918. pid = []
  919. def cbConnected(transport):
  920. pid.append(transport.pid)
  921. # There's now a reap process handler registered.
  922. self.assertIn(transport.pid, process.reapProcessHandlers)
  923. # Kill the process cleanly, triggering an error in the protocol.
  924. transport.loseConnection()
  925. connected.addCallback(cbConnected)
  926. def checkTerminated(ignored):
  927. # The exception was logged.
  928. excs = self.flushLoggedErrors(RuntimeError)
  929. self.assertEqual(len(excs), 1)
  930. # The process is no longer scheduled for reaping.
  931. self.assertNotIn(pid[0], process.reapProcessHandlers)
  932. ended.addCallback(checkTerminated)
  933. return ended
  934. class MockSignal(object):
  935. """
  936. Neuter L{signal.signal}, but pass other attributes unscathed
  937. """
  938. def signal(self, sig, action):
  939. return signal.getsignal(sig)
  940. def __getattr__(self, attr):
  941. return getattr(signal, attr)
  942. class MockOS(object):
  943. """
  944. The mock OS: overwrite L{os}, L{fcntl} and {sys} functions with fake ones.
  945. @ivar exited: set to True when C{_exit} is called.
  946. @type exited: C{bool}
  947. @ivar O_RDWR: dumb value faking C{os.O_RDWR}.
  948. @type O_RDWR: C{int}
  949. @ivar O_NOCTTY: dumb value faking C{os.O_NOCTTY}.
  950. @type O_NOCTTY: C{int}
  951. @ivar WNOHANG: dumb value faking C{os.WNOHANG}.
  952. @type WNOHANG: C{int}
  953. @ivar raiseFork: if not L{None}, subsequent calls to fork will raise this
  954. object.
  955. @type raiseFork: L{None} or C{Exception}
  956. @ivar raiseExec: if set, subsequent calls to execvpe will raise an error.
  957. @type raiseExec: C{bool}
  958. @ivar fdio: fake file object returned by calls to fdopen.
  959. @type fdio: C{BytesIO} or C{BytesIO}
  960. @ivar actions: hold names of some actions executed by the object, in order
  961. of execution.
  962. @type actions: C{list} of C{str}
  963. @ivar closed: keep track of the file descriptor closed.
  964. @type closed: C{list} of C{int}
  965. @ivar child: whether fork return for the child or the parent.
  966. @type child: C{bool}
  967. @ivar pipeCount: count the number of time that C{os.pipe} has been called.
  968. @type pipeCount: C{int}
  969. @ivar raiseWaitPid: if set, subsequent calls to waitpid will raise
  970. the error specified.
  971. @type raiseWaitPid: L{None} or a class
  972. @ivar waitChild: if set, subsequent calls to waitpid will return it.
  973. @type waitChild: L{None} or a tuple
  974. @ivar euid: the uid returned by the fake C{os.geteuid}
  975. @type euid: C{int}
  976. @ivar egid: the gid returned by the fake C{os.getegid}
  977. @type egid: C{int}
  978. @ivar seteuidCalls: stored results of C{os.seteuid} calls.
  979. @type seteuidCalls: C{list}
  980. @ivar setegidCalls: stored results of C{os.setegid} calls.
  981. @type setegidCalls: C{list}
  982. @ivar path: the path returned by C{os.path.expanduser}.
  983. @type path: C{str}
  984. @ivar raiseKill: if set, subsequent call to kill will raise the error
  985. specified.
  986. @type raiseKill: L{None} or an exception instance.
  987. @ivar readData: data returned by C{os.read}.
  988. @type readData: C{str}
  989. """
  990. exited = False
  991. raiseExec = False
  992. fdio = None
  993. child = True
  994. raiseWaitPid = None
  995. raiseFork = None
  996. waitChild = None
  997. euid = 0
  998. egid = 0
  999. path = None
  1000. raiseKill = None
  1001. readData = b""
  1002. def __init__(self):
  1003. """
  1004. Initialize data structures.
  1005. """
  1006. self.actions = []
  1007. self.closed = []
  1008. self.pipeCount = 0
  1009. self.O_RDWR = -1
  1010. self.O_NOCTTY = -2
  1011. self.WNOHANG = -4
  1012. self.WEXITSTATUS = lambda x: 0
  1013. self.WIFEXITED = lambda x: 1
  1014. self.seteuidCalls = []
  1015. self.setegidCalls = []
  1016. def open(self, dev, flags):
  1017. """
  1018. Fake C{os.open}. Return a non fd number to be sure it's not used
  1019. elsewhere.
  1020. """
  1021. return -3
  1022. def fstat(self, fd):
  1023. """
  1024. Fake C{os.fstat}. Return a C{os.stat_result} filled with garbage.
  1025. """
  1026. return os.stat_result((0,) * 10)
  1027. def fdopen(self, fd, flag):
  1028. """
  1029. Fake C{os.fdopen}. Return a file-like object whose content can
  1030. be tested later via C{self.fdio}.
  1031. """
  1032. if flag == "wb":
  1033. self.fdio = BytesIO()
  1034. else:
  1035. assert False
  1036. return self.fdio
  1037. def setsid(self):
  1038. """
  1039. Fake C{os.setsid}. Save action.
  1040. """
  1041. self.actions.append('setsid')
  1042. def fork(self):
  1043. """
  1044. Fake C{os.fork}. Save the action in C{self.actions}, and return 0 if
  1045. C{self.child} is set, or a dumb number.
  1046. """
  1047. self.actions.append(('fork', gc.isenabled()))
  1048. if self.raiseFork is not None:
  1049. raise self.raiseFork
  1050. elif self.child:
  1051. # Child result is 0
  1052. return 0
  1053. else:
  1054. return 21
  1055. def close(self, fd):
  1056. """
  1057. Fake C{os.close}, saving the closed fd in C{self.closed}.
  1058. """
  1059. self.closed.append(fd)
  1060. def dup2(self, fd1, fd2):
  1061. """
  1062. Fake C{os.dup2}. Do nothing.
  1063. """
  1064. def write(self, fd, data):
  1065. """
  1066. Fake C{os.write}. Save action.
  1067. """
  1068. self.actions.append(("write", fd, data))
  1069. def read(self, fd, size):
  1070. """
  1071. Fake C{os.read}: save action, and return C{readData} content.
  1072. @param fd: The file descriptor to read.
  1073. @param size: The maximum number of bytes to read.
  1074. @return: A fixed C{bytes} buffer.
  1075. """
  1076. self.actions.append(('read', fd, size))
  1077. return self.readData
  1078. def execvpe(self, command, args, env):
  1079. """
  1080. Fake C{os.execvpe}. Save the action, and raise an error if
  1081. C{self.raiseExec} is set.
  1082. """
  1083. self.actions.append('exec')
  1084. if self.raiseExec:
  1085. raise RuntimeError("Bar")
  1086. def pipe(self):
  1087. """
  1088. Fake C{os.pipe}. Return non fd numbers to be sure it's not used
  1089. elsewhere, and increment C{self.pipeCount}. This is used to uniquify
  1090. the result.
  1091. """
  1092. self.pipeCount += 1
  1093. return - 2 * self.pipeCount + 1, - 2 * self.pipeCount
  1094. def ttyname(self, fd):
  1095. """
  1096. Fake C{os.ttyname}. Return a dumb string.
  1097. """
  1098. return "foo"
  1099. def _exit(self, code):
  1100. """
  1101. Fake C{os._exit}. Save the action, set the C{self.exited} flag, and
  1102. raise C{SystemError}.
  1103. """
  1104. self.actions.append(('exit', code))
  1105. self.exited = True
  1106. # Don't forget to raise an error, or you'll end up in parent
  1107. # code path.
  1108. raise SystemError()
  1109. def ioctl(self, fd, flags, arg):
  1110. """
  1111. Override C{fcntl.ioctl}. Do nothing.
  1112. """
  1113. def setNonBlocking(self, fd):
  1114. """
  1115. Override C{fdesc.setNonBlocking}. Do nothing.
  1116. """
  1117. def waitpid(self, pid, options):
  1118. """
  1119. Override C{os.waitpid}. Return values meaning that the child process
  1120. has exited, save executed action.
  1121. """
  1122. self.actions.append('waitpid')
  1123. if self.raiseWaitPid is not None:
  1124. raise self.raiseWaitPid
  1125. if self.waitChild is not None:
  1126. return self.waitChild
  1127. return 1, 0
  1128. def settrace(self, arg):
  1129. """
  1130. Override C{sys.settrace} to keep coverage working.
  1131. """
  1132. def getgid(self):
  1133. """
  1134. Override C{os.getgid}. Return a dumb number.
  1135. """
  1136. return 1235
  1137. def getuid(self):
  1138. """
  1139. Override C{os.getuid}. Return a dumb number.
  1140. """
  1141. return 1237
  1142. def setuid(self, val):
  1143. """
  1144. Override C{os.setuid}. Do nothing.
  1145. """
  1146. self.actions.append(('setuid', val))
  1147. def setgid(self, val):
  1148. """
  1149. Override C{os.setgid}. Do nothing.
  1150. """
  1151. self.actions.append(('setgid', val))
  1152. def setregid(self, val1, val2):
  1153. """
  1154. Override C{os.setregid}. Do nothing.
  1155. """
  1156. self.actions.append(('setregid', val1, val2))
  1157. def setreuid(self, val1, val2):
  1158. """
  1159. Override C{os.setreuid}. Save the action.
  1160. """
  1161. self.actions.append(('setreuid', val1, val2))
  1162. def switchUID(self, uid, gid):
  1163. """
  1164. Override C{util.switchuid}. Save the action.
  1165. """
  1166. self.actions.append(('switchuid', uid, gid))
  1167. def openpty(self):
  1168. """
  1169. Override C{pty.openpty}, returning fake file descriptors.
  1170. """
  1171. return -12, -13
  1172. def chdir(self, path):
  1173. """
  1174. Override C{os.chdir}. Save the action.
  1175. @param path: The path to change the current directory to.
  1176. """
  1177. self.actions.append(('chdir', path))
  1178. def geteuid(self):
  1179. """
  1180. Mock C{os.geteuid}, returning C{self.euid} instead.
  1181. """
  1182. return self.euid
  1183. def getegid(self):
  1184. """
  1185. Mock C{os.getegid}, returning C{self.egid} instead.
  1186. """
  1187. return self.egid
  1188. def seteuid(self, egid):
  1189. """
  1190. Mock C{os.seteuid}, store result.
  1191. """
  1192. self.seteuidCalls.append(egid)
  1193. def setegid(self, egid):
  1194. """
  1195. Mock C{os.setegid}, store result.
  1196. """
  1197. self.setegidCalls.append(egid)
  1198. def expanduser(self, path):
  1199. """
  1200. Mock C{os.path.expanduser}.
  1201. """
  1202. return self.path
  1203. def getpwnam(self, user):
  1204. """
  1205. Mock C{pwd.getpwnam}.
  1206. """
  1207. return 0, 0, 1, 2
  1208. def listdir(self, path):
  1209. """
  1210. Override C{os.listdir}, returning fake contents of '/dev/fd'
  1211. """
  1212. return "-1", "-2"
  1213. def kill(self, pid, signalID):
  1214. """
  1215. Override C{os.kill}: save the action and raise C{self.raiseKill} if
  1216. specified.
  1217. """
  1218. self.actions.append(('kill', pid, signalID))
  1219. if self.raiseKill is not None:
  1220. raise self.raiseKill
  1221. def unlink(self, filename):
  1222. """
  1223. Override C{os.unlink}. Save the action.
  1224. @param filename: The file name to remove.
  1225. """
  1226. self.actions.append(('unlink', filename))
  1227. def umask(self, mask):
  1228. """
  1229. Override C{os.umask}. Save the action.
  1230. @param mask: The new file mode creation mask.
  1231. """
  1232. self.actions.append(('umask', mask))
  1233. def getpid(self):
  1234. """
  1235. Return a fixed PID value.
  1236. @return: A fixed value.
  1237. """
  1238. return 6789
  1239. def getfilesystemencoding(self):
  1240. """
  1241. Return a fixed filesystem encoding.
  1242. @return: A fixed value of "utf8".
  1243. """
  1244. return "utf8"
  1245. if process is not None:
  1246. class DumbProcessWriter(process.ProcessWriter):
  1247. """
  1248. A fake L{process.ProcessWriter} used for tests.
  1249. """
  1250. def startReading(self):
  1251. """
  1252. Here's the faking: don't do anything here.
  1253. """
  1254. class DumbProcessReader(process.ProcessReader):
  1255. """
  1256. A fake L{process.ProcessReader} used for tests.
  1257. """
  1258. def startReading(self):
  1259. """
  1260. Here's the faking: don't do anything here.
  1261. """
  1262. class DumbPTYProcess(process.PTYProcess):
  1263. """
  1264. A fake L{process.PTYProcess} used for tests.
  1265. """
  1266. def startReading(self):
  1267. """
  1268. Here's the faking: don't do anything here.
  1269. """
  1270. class MockProcessTests(unittest.TestCase):
  1271. """
  1272. Mock a process runner to test forked child code path.
  1273. """
  1274. if process is None:
  1275. skip = "twisted.internet.process is never used on Windows"
  1276. def setUp(self):
  1277. """
  1278. Replace L{process} os, fcntl, sys, switchUID, fdesc and pty modules
  1279. with the mock class L{MockOS}.
  1280. """
  1281. if gc.isenabled():
  1282. self.addCleanup(gc.enable)
  1283. else:
  1284. self.addCleanup(gc.disable)
  1285. self.mockos = MockOS()
  1286. self.mockos.euid = 1236
  1287. self.mockos.egid = 1234
  1288. self.patch(process, "os", self.mockos)
  1289. self.patch(process, "fcntl", self.mockos)
  1290. self.patch(process, "sys", self.mockos)
  1291. self.patch(process, "switchUID", self.mockos.switchUID)
  1292. self.patch(process, "fdesc", self.mockos)
  1293. self.patch(process.Process, "processReaderFactory", DumbProcessReader)
  1294. self.patch(process.Process, "processWriterFactory", DumbProcessWriter)
  1295. self.patch(process, "pty", self.mockos)
  1296. self.mocksig = MockSignal()
  1297. self.patch(process, "signal", self.mocksig)
  1298. def tearDown(self):
  1299. """
  1300. Reset processes registered for reap.
  1301. """
  1302. process.reapProcessHandlers = {}
  1303. def test_mockFork(self):
  1304. """
  1305. Test a classic spawnProcess. Check the path of the client code:
  1306. fork, exec, exit.
  1307. """
  1308. gc.enable()
  1309. cmd = b'/mock/ouch'
  1310. d = defer.Deferred()
  1311. p = TrivialProcessProtocol(d)
  1312. try:
  1313. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1314. usePTY=False)
  1315. except SystemError:
  1316. self.assertTrue(self.mockos.exited)
  1317. self.assertEqual(
  1318. self.mockos.actions, [("fork", False), "exec", ("exit", 1)])
  1319. else:
  1320. self.fail("Should not be here")
  1321. # It should leave the garbage collector disabled.
  1322. self.assertFalse(gc.isenabled())
  1323. def _mockForkInParentTest(self):
  1324. """
  1325. Assert that in the main process, spawnProcess disables the garbage
  1326. collector, calls fork, closes the pipe file descriptors it created for
  1327. the child process, and calls waitpid.
  1328. """
  1329. self.mockos.child = False
  1330. cmd = b'/mock/ouch'
  1331. d = defer.Deferred()
  1332. p = TrivialProcessProtocol(d)
  1333. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1334. usePTY=False)
  1335. # It should close the first read pipe, and the 2 last write pipes
  1336. self.assertEqual(set(self.mockos.closed), set([-1, -4, -6]))
  1337. self.assertEqual(self.mockos.actions, [("fork", False), "waitpid"])
  1338. def test_mockForkInParentGarbageCollectorEnabled(self):
  1339. """
  1340. The garbage collector should be enabled when L{reactor.spawnProcess}
  1341. returns if it was initially enabled.
  1342. @see L{_mockForkInParentTest}
  1343. """
  1344. gc.enable()
  1345. self._mockForkInParentTest()
  1346. self.assertTrue(gc.isenabled())
  1347. def test_mockForkInParentGarbageCollectorDisabled(self):
  1348. """
  1349. The garbage collector should be disabled when L{reactor.spawnProcess}
  1350. returns if it was initially disabled.
  1351. @see L{_mockForkInParentTest}
  1352. """
  1353. gc.disable()
  1354. self._mockForkInParentTest()
  1355. self.assertFalse(gc.isenabled())
  1356. def test_mockForkTTY(self):
  1357. """
  1358. Test a TTY spawnProcess: check the path of the client code:
  1359. fork, exec, exit.
  1360. """
  1361. cmd = b'/mock/ouch'
  1362. d = defer.Deferred()
  1363. p = TrivialProcessProtocol(d)
  1364. self.assertRaises(SystemError, reactor.spawnProcess, p, cmd, [b'ouch'],
  1365. env=None, usePTY=True)
  1366. self.assertTrue(self.mockos.exited)
  1367. self.assertEqual(
  1368. self.mockos.actions,
  1369. [("fork", False), "setsid", "exec", ("exit", 1)])
  1370. def _mockWithForkError(self):
  1371. """
  1372. Assert that if the fork call fails, no other process setup calls are
  1373. made and that spawnProcess raises the exception fork raised.
  1374. """
  1375. self.mockos.raiseFork = OSError(errno.EAGAIN, None)
  1376. protocol = TrivialProcessProtocol(None)
  1377. self.assertRaises(OSError, reactor.spawnProcess, protocol, None)
  1378. self.assertEqual(self.mockos.actions, [("fork", False)])
  1379. def test_mockWithForkErrorGarbageCollectorEnabled(self):
  1380. """
  1381. The garbage collector should be enabled when L{reactor.spawnProcess}
  1382. raises because L{os.fork} raised, if it was initially enabled.
  1383. """
  1384. gc.enable()
  1385. self._mockWithForkError()
  1386. self.assertTrue(gc.isenabled())
  1387. def test_mockWithForkErrorGarbageCollectorDisabled(self):
  1388. """
  1389. The garbage collector should be disabled when
  1390. L{reactor.spawnProcess} raises because L{os.fork} raised, if it was
  1391. initially disabled.
  1392. """
  1393. gc.disable()
  1394. self._mockWithForkError()
  1395. self.assertFalse(gc.isenabled())
  1396. def test_mockForkErrorCloseFDs(self):
  1397. """
  1398. When C{os.fork} raises an exception, the file descriptors created
  1399. before are closed and don't leak.
  1400. """
  1401. self._mockWithForkError()
  1402. self.assertEqual(set(self.mockos.closed), set([-1, -4, -6, -2, -3, -5]))
  1403. def test_mockForkErrorGivenFDs(self):
  1404. """
  1405. When C{os.forks} raises an exception and that file descriptors have
  1406. been specified with the C{childFDs} arguments of
  1407. L{reactor.spawnProcess}, they are not closed.
  1408. """
  1409. self.mockos.raiseFork = OSError(errno.EAGAIN, None)
  1410. protocol = TrivialProcessProtocol(None)
  1411. self.assertRaises(OSError, reactor.spawnProcess, protocol, None,
  1412. childFDs={0: -10, 1: -11, 2: -13})
  1413. self.assertEqual(self.mockos.actions, [("fork", False)])
  1414. self.assertEqual(self.mockos.closed, [])
  1415. # We can also put "r" or "w" to let twisted create the pipes
  1416. self.assertRaises(OSError, reactor.spawnProcess, protocol, None,
  1417. childFDs={0: "r", 1: -11, 2: -13})
  1418. self.assertEqual(set(self.mockos.closed), set([-1, -2]))
  1419. def test_mockForkErrorClosePTY(self):
  1420. """
  1421. When C{os.fork} raises an exception, the file descriptors created by
  1422. C{pty.openpty} are closed and don't leak, when C{usePTY} is set to
  1423. C{True}.
  1424. """
  1425. self.mockos.raiseFork = OSError(errno.EAGAIN, None)
  1426. protocol = TrivialProcessProtocol(None)
  1427. self.assertRaises(OSError, reactor.spawnProcess, protocol, None,
  1428. usePTY=True)
  1429. self.assertEqual(self.mockos.actions, [("fork", False)])
  1430. self.assertEqual(set(self.mockos.closed), set([-12, -13]))
  1431. def test_mockForkErrorPTYGivenFDs(self):
  1432. """
  1433. If a tuple is passed to C{usePTY} to specify slave and master file
  1434. descriptors and that C{os.fork} raises an exception, these file
  1435. descriptors aren't closed.
  1436. """
  1437. self.mockos.raiseFork = OSError(errno.EAGAIN, None)
  1438. protocol = TrivialProcessProtocol(None)
  1439. self.assertRaises(OSError, reactor.spawnProcess, protocol, None,
  1440. usePTY=(-20, -21, 'foo'))
  1441. self.assertEqual(self.mockos.actions, [("fork", False)])
  1442. self.assertEqual(self.mockos.closed, [])
  1443. def test_mockWithExecError(self):
  1444. """
  1445. Spawn a process but simulate an error during execution in the client
  1446. path: C{os.execvpe} raises an error. It should close all the standard
  1447. fds, try to print the error encountered, and exit cleanly.
  1448. """
  1449. cmd = b'/mock/ouch'
  1450. d = defer.Deferred()
  1451. p = TrivialProcessProtocol(d)
  1452. self.mockos.raiseExec = True
  1453. try:
  1454. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1455. usePTY=False)
  1456. except SystemError:
  1457. self.assertTrue(self.mockos.exited)
  1458. self.assertEqual(
  1459. self.mockos.actions, [("fork", False), "exec", ("exit", 1)])
  1460. # Check that fd have been closed
  1461. self.assertIn(0, self.mockos.closed)
  1462. self.assertIn(1, self.mockos.closed)
  1463. self.assertIn(2, self.mockos.closed)
  1464. # Check content of traceback
  1465. self.assertIn(b"RuntimeError: Bar", self.mockos.fdio.getvalue())
  1466. else:
  1467. self.fail("Should not be here")
  1468. def test_mockSetUid(self):
  1469. """
  1470. Try creating a process with setting its uid: it's almost the same path
  1471. as the standard path, but with a C{switchUID} call before the exec.
  1472. """
  1473. cmd = b'/mock/ouch'
  1474. d = defer.Deferred()
  1475. p = TrivialProcessProtocol(d)
  1476. try:
  1477. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1478. usePTY=False, uid=8080)
  1479. except SystemError:
  1480. self.assertTrue(self.mockos.exited)
  1481. self.assertEqual(
  1482. self.mockos.actions,
  1483. [('fork', False), ('setuid', 0), ('setgid', 0),
  1484. ('switchuid', 8080, 1234), 'exec', ('exit', 1)])
  1485. else:
  1486. self.fail("Should not be here")
  1487. def test_mockSetUidInParent(self):
  1488. """
  1489. When spawning a child process with a UID different from the UID of the
  1490. current process, the current process does not have its UID changed.
  1491. """
  1492. self.mockos.child = False
  1493. cmd = b'/mock/ouch'
  1494. d = defer.Deferred()
  1495. p = TrivialProcessProtocol(d)
  1496. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1497. usePTY=False, uid=8080)
  1498. self.assertEqual(self.mockos.actions, [('fork', False), 'waitpid'])
  1499. def test_mockPTYSetUid(self):
  1500. """
  1501. Try creating a PTY process with setting its uid: it's almost the same
  1502. path as the standard path, but with a C{switchUID} call before the
  1503. exec.
  1504. """
  1505. cmd = b'/mock/ouch'
  1506. d = defer.Deferred()
  1507. p = TrivialProcessProtocol(d)
  1508. try:
  1509. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1510. usePTY=True, uid=8081)
  1511. except SystemError:
  1512. self.assertTrue(self.mockos.exited)
  1513. self.assertEqual(
  1514. self.mockos.actions,
  1515. [('fork', False), 'setsid', ('setuid', 0), ('setgid', 0),
  1516. ('switchuid', 8081, 1234), 'exec', ('exit', 1)])
  1517. else:
  1518. self.fail("Should not be here")
  1519. def test_mockPTYSetUidInParent(self):
  1520. """
  1521. When spawning a child process with PTY and a UID different from the UID
  1522. of the current process, the current process does not have its UID
  1523. changed.
  1524. """
  1525. self.mockos.child = False
  1526. cmd = b'/mock/ouch'
  1527. d = defer.Deferred()
  1528. p = TrivialProcessProtocol(d)
  1529. oldPTYProcess = process.PTYProcess
  1530. try:
  1531. process.PTYProcess = DumbPTYProcess
  1532. reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1533. usePTY=True, uid=8080)
  1534. finally:
  1535. process.PTYProcess = oldPTYProcess
  1536. self.assertEqual(self.mockos.actions, [('fork', False), 'waitpid'])
  1537. def test_mockWithWaitError(self):
  1538. """
  1539. Test that reapProcess logs errors raised.
  1540. """
  1541. self.mockos.child = False
  1542. cmd = b'/mock/ouch'
  1543. self.mockos.waitChild = (0, 0)
  1544. d = defer.Deferred()
  1545. p = TrivialProcessProtocol(d)
  1546. proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1547. usePTY=False)
  1548. self.assertEqual(self.mockos.actions, [("fork", False), "waitpid"])
  1549. self.mockos.raiseWaitPid = OSError()
  1550. proc.reapProcess()
  1551. errors = self.flushLoggedErrors()
  1552. self.assertEqual(len(errors), 1)
  1553. errors[0].trap(OSError)
  1554. def test_mockErrorECHILDInReapProcess(self):
  1555. """
  1556. Test that reapProcess doesn't log anything when waitpid raises a
  1557. C{OSError} with errno C{ECHILD}.
  1558. """
  1559. self.mockos.child = False
  1560. cmd = b'/mock/ouch'
  1561. self.mockos.waitChild = (0, 0)
  1562. d = defer.Deferred()
  1563. p = TrivialProcessProtocol(d)
  1564. proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None,
  1565. usePTY=False)
  1566. self.assertEqual(self.mockos.actions, [("fork", False), "waitpid"])
  1567. self.mockos.raiseWaitPid = OSError()
  1568. self.mockos.raiseWaitPid.errno = errno.ECHILD
  1569. # This should not produce any errors
  1570. proc.reapProcess()
  1571. def test_mockErrorInPipe(self):
  1572. """
  1573. If C{os.pipe} raises an exception after some pipes where created, the
  1574. created pipes are closed and don't leak.
  1575. """
  1576. pipes = [-1, -2, -3, -4]
  1577. def pipe():
  1578. try:
  1579. return pipes.pop(0), pipes.pop(0)
  1580. except IndexError:
  1581. raise OSError()
  1582. self.mockos.pipe = pipe
  1583. protocol = TrivialProcessProtocol(None)
  1584. self.assertRaises(OSError, reactor.spawnProcess, protocol, None)
  1585. self.assertEqual(self.mockos.actions, [])
  1586. self.assertEqual(set(self.mockos.closed), set([-4, -3, -2, -1]))
  1587. def test_kill(self):
  1588. """
  1589. L{process.Process.signalProcess} calls C{os.kill} translating the given
  1590. signal string to the PID.
  1591. """
  1592. self.mockos.child = False
  1593. self.mockos.waitChild = (0, 0)
  1594. cmd = b'/mock/ouch'
  1595. p = TrivialProcessProtocol(None)
  1596. proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None, usePTY=False)
  1597. proc.signalProcess("KILL")
  1598. self.assertEqual(self.mockos.actions,
  1599. [('fork', False), 'waitpid', ('kill', 21, signal.SIGKILL)])
  1600. def test_killExited(self):
  1601. """
  1602. L{process.Process.signalProcess} raises L{error.ProcessExitedAlready}
  1603. if the process has exited.
  1604. """
  1605. self.mockos.child = False
  1606. cmd = b'/mock/ouch'
  1607. p = TrivialProcessProtocol(None)
  1608. proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None, usePTY=False)
  1609. # We didn't specify a waitpid value, so the waitpid call in
  1610. # registerReapProcessHandler has already reaped the process
  1611. self.assertRaises(error.ProcessExitedAlready,
  1612. proc.signalProcess, "KILL")
  1613. def test_killExitedButNotDetected(self):
  1614. """
  1615. L{process.Process.signalProcess} raises L{error.ProcessExitedAlready}
  1616. if the process has exited but that twisted hasn't seen it (for example,
  1617. if the process has been waited outside of twisted): C{os.kill} then
  1618. raise C{OSError} with C{errno.ESRCH} as errno.
  1619. """
  1620. self.mockos.child = False
  1621. self.mockos.waitChild = (0, 0)
  1622. cmd = b'/mock/ouch'
  1623. p = TrivialProcessProtocol(None)
  1624. proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None, usePTY=False)
  1625. self.mockos.raiseKill = OSError(errno.ESRCH, "Not found")
  1626. self.assertRaises(error.ProcessExitedAlready,
  1627. proc.signalProcess, "KILL")
  1628. def test_killErrorInKill(self):
  1629. """
  1630. L{process.Process.signalProcess} doesn't mask C{OSError} exceptions if
  1631. the errno is different from C{errno.ESRCH}.
  1632. """
  1633. self.mockos.child = False
  1634. self.mockos.waitChild = (0, 0)
  1635. cmd = b'/mock/ouch'
  1636. p = TrivialProcessProtocol(None)
  1637. proc = reactor.spawnProcess(p, cmd, [b'ouch'], env=None, usePTY=False)
  1638. self.mockos.raiseKill = OSError(errno.EINVAL, "Invalid signal")
  1639. err = self.assertRaises(OSError,
  1640. proc.signalProcess, "KILL")
  1641. self.assertEqual(err.errno, errno.EINVAL)
  1642. class PosixProcessTests(unittest.TestCase, PosixProcessBase):
  1643. # add two non-pty test cases
  1644. def test_stderr(self):
  1645. """
  1646. Bytes written to stderr by the spawned process are passed to the
  1647. C{errReceived} callback on the C{ProcessProtocol} passed to
  1648. C{spawnProcess}.
  1649. """
  1650. value = "42"
  1651. p = Accumulator()
  1652. d = p.endedDeferred = defer.Deferred()
  1653. reactor.spawnProcess(p, pyExe,
  1654. [pyExe, b"-c",
  1655. networkString("import sys; sys.stderr.write"
  1656. "('{0}')".format(value))],
  1657. env=None, path="/tmp",
  1658. usePTY=self.usePTY)
  1659. def processEnded(ign):
  1660. self.assertEqual(b"42", p.errF.getvalue())
  1661. return d.addCallback(processEnded)
  1662. def test_process(self):
  1663. cmd = self.getCommand('gzip')
  1664. s = b"there's no place like home!\n" * 3
  1665. p = Accumulator()
  1666. d = p.endedDeferred = defer.Deferred()
  1667. reactor.spawnProcess(p, cmd, [cmd, b"-c"], env=None, path="/tmp",
  1668. usePTY=self.usePTY)
  1669. p.transport.write(s)
  1670. p.transport.closeStdin()
  1671. def processEnded(ign):
  1672. f = p.outF
  1673. f.seek(0, 0)
  1674. with gzip.GzipFile(fileobj=f) as gf:
  1675. self.assertEqual(gf.read(), s)
  1676. return d.addCallback(processEnded)
  1677. class PosixProcessPTYTests(unittest.TestCase, PosixProcessBase):
  1678. """
  1679. Just like PosixProcessTests, but use ptys instead of pipes.
  1680. """
  1681. usePTY = True
  1682. # PTYs only offer one input and one output. What still makes sense?
  1683. # testNormalTermination
  1684. # test_abnormalTermination
  1685. # testSignal
  1686. # testProcess, but not without p.transport.closeStdin
  1687. # might be solveable: TODO: add test if so
  1688. def test_openingTTY(self):
  1689. scriptPath = b"twisted.test.process_tty"
  1690. p = Accumulator()
  1691. d = p.endedDeferred = defer.Deferred()
  1692. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
  1693. env=properEnv, usePTY=self.usePTY)
  1694. p.transport.write(b"hello world!\n")
  1695. def processEnded(ign):
  1696. self.assertRaises(
  1697. error.ProcessExitedAlready, p.transport.signalProcess, 'HUP')
  1698. self.assertEqual(
  1699. p.outF.getvalue(),
  1700. b"hello world!\r\nhello world!\r\n",
  1701. ("Error message from process_tty "
  1702. "follows:\n\n%s\n\n" % (p.outF.getvalue(),)))
  1703. return d.addCallback(processEnded)
  1704. def test_badArgs(self):
  1705. pyArgs = [pyExe, b"-u", b"-c", b"print('hello')"]
  1706. p = Accumulator()
  1707. self.assertRaises(ValueError, reactor.spawnProcess, p, pyExe, pyArgs,
  1708. usePTY=1, childFDs={1:b'r'})
  1709. class Win32SignalProtocol(SignalProtocol):
  1710. """
  1711. A win32-specific process protocol that handles C{processEnded}
  1712. differently: processes should exit with exit code 1.
  1713. """
  1714. def processEnded(self, reason):
  1715. """
  1716. Callback C{self.deferred} with L{None} if C{reason} is a
  1717. L{error.ProcessTerminated} failure with C{exitCode} set to 1.
  1718. Otherwise, errback with a C{ValueError} describing the problem.
  1719. """
  1720. if not reason.check(error.ProcessTerminated):
  1721. return self.deferred.errback(
  1722. ValueError("wrong termination: %s" % (reason,)))
  1723. v = reason.value
  1724. if v.exitCode != 1:
  1725. return self.deferred.errback(
  1726. ValueError("Wrong exit code: %s" % (v.exitCode,)))
  1727. self.deferred.callback(None)
  1728. class Win32ProcessTests(unittest.TestCase):
  1729. """
  1730. Test process programs that are packaged with twisted.
  1731. """
  1732. def test_stdinReader(self):
  1733. scriptPath = b"twisted.test.process_stdinreader"
  1734. p = Accumulator()
  1735. d = p.endedDeferred = defer.Deferred()
  1736. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
  1737. env=properEnv)
  1738. p.transport.write(b"hello, world")
  1739. p.transport.closeStdin()
  1740. def processEnded(ign):
  1741. self.assertEqual(p.errF.getvalue(), b"err\nerr\n")
  1742. self.assertEqual(p.outF.getvalue(), b"out\nhello, world\nout\n")
  1743. return d.addCallback(processEnded)
  1744. def test_badArgs(self):
  1745. pyArgs = [pyExe, b"-u", b"-c", b"print('hello')"]
  1746. p = Accumulator()
  1747. self.assertRaises(ValueError,
  1748. reactor.spawnProcess, p, pyExe, pyArgs, uid=1)
  1749. self.assertRaises(ValueError,
  1750. reactor.spawnProcess, p, pyExe, pyArgs, gid=1)
  1751. self.assertRaises(ValueError,
  1752. reactor.spawnProcess, p, pyExe, pyArgs, usePTY=1)
  1753. self.assertRaises(ValueError,
  1754. reactor.spawnProcess, p, pyExe, pyArgs, childFDs={1:'r'})
  1755. def _testSignal(self, sig):
  1756. scriptPath = b"twisted.test.process_signal"
  1757. d = defer.Deferred()
  1758. p = Win32SignalProtocol(d, sig)
  1759. reactor.spawnProcess(p, pyExe, [pyExe, b"-u", b"-m", scriptPath],
  1760. env=properEnv)
  1761. return d
  1762. def test_signalTERM(self):
  1763. """
  1764. Sending the SIGTERM signal terminates a created process, and
  1765. C{processEnded} is called with a L{error.ProcessTerminated} instance
  1766. with the C{exitCode} attribute set to 1.
  1767. """
  1768. return self._testSignal('TERM')
  1769. def test_signalINT(self):
  1770. """
  1771. Sending the SIGINT signal terminates a created process, and
  1772. C{processEnded} is called with a L{error.ProcessTerminated} instance
  1773. with the C{exitCode} attribute set to 1.
  1774. """
  1775. return self._testSignal('INT')
  1776. def test_signalKILL(self):
  1777. """
  1778. Sending the SIGKILL signal terminates a created process, and
  1779. C{processEnded} is called with a L{error.ProcessTerminated} instance
  1780. with the C{exitCode} attribute set to 1.
  1781. """
  1782. return self._testSignal('KILL')
  1783. def test_closeHandles(self):
  1784. """
  1785. The win32 handles should be properly closed when the process exits.
  1786. """
  1787. import win32api
  1788. connected = defer.Deferred()
  1789. ended = defer.Deferred()
  1790. class SimpleProtocol(protocol.ProcessProtocol):
  1791. """
  1792. A protocol that fires deferreds when connected and disconnected.
  1793. """
  1794. def makeConnection(self, transport):
  1795. connected.callback(transport)
  1796. def processEnded(self, reason):
  1797. ended.callback(None)
  1798. p = SimpleProtocol()
  1799. pyArgs = [pyExe, b"-u", b"-c", b"print('hello')"]
  1800. proc = reactor.spawnProcess(p, pyExe, pyArgs)
  1801. def cbConnected(transport):
  1802. self.assertIs(transport, proc)
  1803. # perform a basic validity test on the handles
  1804. win32api.GetHandleInformation(proc.hProcess)
  1805. win32api.GetHandleInformation(proc.hThread)
  1806. # And save their values for later
  1807. self.hProcess = proc.hProcess
  1808. self.hThread = proc.hThread
  1809. connected.addCallback(cbConnected)
  1810. def checkTerminated(ignored):
  1811. # The attributes on the process object must be reset...
  1812. self.assertIsNone(proc.pid)
  1813. self.assertIsNone(proc.hProcess)
  1814. self.assertIsNone(proc.hThread)
  1815. # ...and the handles must be closed.
  1816. self.assertRaises(win32api.error,
  1817. win32api.GetHandleInformation, self.hProcess)
  1818. self.assertRaises(win32api.error,
  1819. win32api.GetHandleInformation, self.hThread)
  1820. ended.addCallback(checkTerminated)
  1821. return defer.gatherResults([connected, ended])
  1822. class Win32UnicodeEnvironmentTests(unittest.TestCase):
  1823. """
  1824. Tests for Unicode environment on Windows
  1825. """
  1826. goodKey = u'UNICODE'
  1827. goodValue = u'UNICODE'
  1828. def test_encodableUnicodeEnvironment(self):
  1829. """
  1830. Test C{os.environ} (inherited by every subprocess on Windows) that
  1831. contains an ascii-encodable Unicode string. This is different from
  1832. passing Unicode environment explicitly to spawnProcess (which is not
  1833. supported on Python 2).
  1834. """
  1835. os.environ[self.goodKey] = self.goodValue
  1836. self.addCleanup(operator.delitem, os.environ, self.goodKey)
  1837. p = GetEnvironmentDictionary.run(reactor, [], properEnv)
  1838. def gotEnvironment(environ):
  1839. self.assertEqual(
  1840. environ[self.goodKey.encode('ascii')],
  1841. self.goodValue.encode('ascii'))
  1842. return p.getResult().addCallback(gotEnvironment)
  1843. class DumbWin32ProcTests(unittest.TestCase):
  1844. """
  1845. L{twisted.internet._dumbwin32proc} tests.
  1846. """
  1847. def test_pid(self):
  1848. """
  1849. Simple test for the pid attribute of Process on win32.
  1850. Launch process with mock win32process. The only mock aspect of this
  1851. module is that the pid of the process created will always be 42.
  1852. """
  1853. from twisted.internet import _dumbwin32proc
  1854. from twisted.test import mock_win32process
  1855. self.patch(_dumbwin32proc, "win32process", mock_win32process)
  1856. scriptPath = FilePath(__file__).sibling(u"process_cmdline.py").path
  1857. pyExe = FilePath(sys.executable).asTextMode().path
  1858. d = defer.Deferred()
  1859. processProto = TrivialProcessProtocol(d)
  1860. comspec = u"cmd.exe"
  1861. cmd = [comspec, u"/c", pyExe, scriptPath]
  1862. p = _dumbwin32proc.Process(
  1863. reactor, processProto, None, cmd, {}, None)
  1864. self.assertEqual(42, p.pid)
  1865. self.assertEqual("<Process pid=42>", repr(p))
  1866. def pidCompleteCb(result):
  1867. self.assertIsNone(p.pid)
  1868. return d.addCallback(pidCompleteCb)
  1869. def test_findShebang(self):
  1870. """
  1871. Look for the string after the shebang C{#!}
  1872. in a file.
  1873. """
  1874. from twisted.internet._dumbwin32proc import _findShebang
  1875. cgiScript = FilePath(b"example.cgi")
  1876. cgiScript.setContent(b"#!/usr/bin/python")
  1877. program = _findShebang(cgiScript.path)
  1878. self.assertEqual(program, "/usr/bin/python")
  1879. class Win32CreateProcessFlagsTests(unittest.TestCase):
  1880. """
  1881. Check the flags passed to CreateProcess.
  1882. """
  1883. @defer.inlineCallbacks
  1884. def test_flags(self):
  1885. """
  1886. Verify that the flags passed to win32process.CreateProcess() prevent a
  1887. new console window from being created. Use the following script
  1888. to test this interactively::
  1889. # Add the following lines to a script named
  1890. # should_not_open_console.pyw
  1891. from twisted.internet import reactor, utils
  1892. def write_result(result):
  1893. open("output.log", "w").write(repr(result))
  1894. reactor.stop()
  1895. PING_EXE = r"c:\windows\system32\ping.exe"
  1896. d = utils.getProcessOutput(PING_EXE, ["slashdot.org"])
  1897. d.addCallbacks(write_result)
  1898. reactor.run()
  1899. To test this, run::
  1900. pythonw.exe should_not_open_console.pyw
  1901. """
  1902. from twisted.internet import _dumbwin32proc
  1903. flags = []
  1904. realCreateProcess = _dumbwin32proc.win32process.CreateProcess
  1905. def fakeCreateprocess(appName, commandLine, processAttributes,
  1906. threadAttributes, bInheritHandles, creationFlags,
  1907. newEnvironment, currentDirectory, startupinfo):
  1908. """
  1909. See the Windows API documentation for I{CreateProcess} for further details.
  1910. @param appName: The name of the module to be executed
  1911. @param commandLine: The command line to be executed.
  1912. @param processAttributes: Pointer to SECURITY_ATTRIBUTES structure or None.
  1913. @param threadAttributes: Pointer to SECURITY_ATTRIBUTES structure or None
  1914. @param bInheritHandles: boolean to determine if inheritable handles from this
  1915. process are inherited in the new process
  1916. @param creationFlags: flags that control priority flags and creation of process.
  1917. @param newEnvironment: pointer to new environment block for new process, or None.
  1918. @param currentDirectory: full path to current directory of new process.
  1919. @param startupinfo: Pointer to STARTUPINFO or STARTUPINFOEX structure
  1920. @return: True on success, False on failure
  1921. @rtype: L{bool}
  1922. """
  1923. flags.append(creationFlags)
  1924. return realCreateProcess(appName, commandLine,
  1925. processAttributes, threadAttributes,
  1926. bInheritHandles, creationFlags, newEnvironment,
  1927. currentDirectory, startupinfo)
  1928. self.patch(_dumbwin32proc.win32process, "CreateProcess",
  1929. fakeCreateprocess)
  1930. exe = sys.executable
  1931. scriptPath = FilePath(__file__).sibling("process_cmdline.py")
  1932. d = defer.Deferred()
  1933. processProto = TrivialProcessProtocol(d)
  1934. comspec = str(os.environ["COMSPEC"])
  1935. cmd = [comspec, "/c", exe, scriptPath.path]
  1936. _dumbwin32proc.Process(reactor, processProto, None, cmd, {}, None)
  1937. yield d
  1938. self.assertEqual(flags,
  1939. [_dumbwin32proc.win32process.CREATE_NO_WINDOW])
  1940. class UtilTests(unittest.TestCase):
  1941. """
  1942. Tests for process-related helper functions (currently only
  1943. L{procutils.which}.
  1944. """
  1945. def setUp(self):
  1946. """
  1947. Create several directories and files, some of which are executable
  1948. and some of which are not. Save the current PATH setting.
  1949. """
  1950. j = os.path.join
  1951. base = self.mktemp()
  1952. self.foo = j(base, "foo")
  1953. self.baz = j(base, "baz")
  1954. self.foobar = j(self.foo, "bar")
  1955. self.foobaz = j(self.foo, "baz")
  1956. self.bazfoo = j(self.baz, "foo")
  1957. self.bazbar = j(self.baz, "bar")
  1958. for d in self.foobar, self.foobaz, self.bazfoo, self.bazbar:
  1959. os.makedirs(d)
  1960. for name, mode in [(j(self.foobaz, "executable"), 0o700),
  1961. (j(self.foo, "executable"), 0o700),
  1962. (j(self.bazfoo, "executable"), 0o700),
  1963. (j(self.bazfoo, "executable.bin"), 0o700),
  1964. (j(self.bazbar, "executable"), 0)]:
  1965. open(name, "wb").close()
  1966. os.chmod(name, mode)
  1967. self.oldPath = os.environ.get('PATH', None)
  1968. os.environ['PATH'] = os.pathsep.join((
  1969. self.foobar, self.foobaz, self.bazfoo, self.bazbar))
  1970. def tearDown(self):
  1971. """
  1972. Restore the saved PATH setting, and set all created files readable
  1973. again so that they can be deleted easily.
  1974. """
  1975. os.chmod(os.path.join(self.bazbar, "executable"), stat.S_IWUSR)
  1976. if self.oldPath is None:
  1977. try:
  1978. del os.environ['PATH']
  1979. except KeyError:
  1980. pass
  1981. else:
  1982. os.environ['PATH'] = self.oldPath
  1983. def test_whichWithoutPATH(self):
  1984. """
  1985. Test that if C{os.environ} does not have a C{'PATH'} key,
  1986. L{procutils.which} returns an empty list.
  1987. """
  1988. del os.environ['PATH']
  1989. self.assertEqual(procutils.which("executable"), [])
  1990. def test_which(self):
  1991. j = os.path.join
  1992. paths = procutils.which("executable")
  1993. expectedPaths = [j(self.foobaz, "executable"),
  1994. j(self.bazfoo, "executable")]
  1995. if runtime.platform.isWindows():
  1996. expectedPaths.append(j(self.bazbar, "executable"))
  1997. self.assertEqual(paths, expectedPaths)
  1998. def test_whichPathExt(self):
  1999. j = os.path.join
  2000. old = os.environ.get('PATHEXT', None)
  2001. os.environ['PATHEXT'] = os.pathsep.join(('.bin', '.exe', '.sh'))
  2002. try:
  2003. paths = procutils.which("executable")
  2004. finally:
  2005. if old is None:
  2006. del os.environ['PATHEXT']
  2007. else:
  2008. os.environ['PATHEXT'] = old
  2009. expectedPaths = [j(self.foobaz, "executable"),
  2010. j(self.bazfoo, "executable"),
  2011. j(self.bazfoo, "executable.bin")]
  2012. if runtime.platform.isWindows():
  2013. expectedPaths.append(j(self.bazbar, "executable"))
  2014. self.assertEqual(paths, expectedPaths)
  2015. class ClosingPipesProcessProtocol(protocol.ProcessProtocol):
  2016. output = b''
  2017. errput = b''
  2018. def __init__(self, outOrErr):
  2019. self.deferred = defer.Deferred()
  2020. self.outOrErr = outOrErr
  2021. def processEnded(self, reason):
  2022. self.deferred.callback(reason)
  2023. def outReceived(self, data):
  2024. self.output += data
  2025. def errReceived(self, data):
  2026. self.errput += data
  2027. class ClosingPipesTests(unittest.TestCase):
  2028. def doit(self, fd):
  2029. """
  2030. Create a child process and close one of its output descriptors using
  2031. L{IProcessTransport.closeStdout} or L{IProcessTransport.closeStderr}.
  2032. Return a L{Deferred} which fires after verifying that the descriptor was
  2033. really closed.
  2034. """
  2035. p = ClosingPipesProcessProtocol(True)
  2036. self.assertFailure(p.deferred, error.ProcessTerminated)
  2037. p.deferred.addCallback(self._endProcess, p)
  2038. reactor.spawnProcess(
  2039. p, pyExe, [
  2040. pyExe, b'-u', b'-c',
  2041. networkString('try: input = raw_input\n'
  2042. 'except NameError: pass\n'
  2043. 'input()\n'
  2044. 'import sys, os, time\n'
  2045. # Give the system a bit of time to notice the closed
  2046. # descriptor. Another option would be to poll() for HUP
  2047. # instead of relying on an os.write to fail with SIGPIPE.
  2048. # However, that wouldn't work on OS X (or Windows?).
  2049. 'for i in range(1000):\n'
  2050. ' os.write(%d, b"foo\\n")\n'
  2051. ' time.sleep(0.01)\n'
  2052. 'sys.exit(42)\n' % (fd,))
  2053. ],
  2054. env=None)
  2055. if fd == 1:
  2056. p.transport.closeStdout()
  2057. elif fd == 2:
  2058. p.transport.closeStderr()
  2059. else:
  2060. raise RuntimeError
  2061. # Give the close time to propagate
  2062. p.transport.write(b'go\n')
  2063. # make the buggy case not hang
  2064. p.transport.closeStdin()
  2065. return p.deferred
  2066. def _endProcess(self, reason, p):
  2067. """
  2068. Check that a failed write prevented the process from getting to its
  2069. custom exit code.
  2070. """
  2071. # child must not get past that write without raising
  2072. self.assertNotEqual(
  2073. reason.exitCode, 42, 'process reason was %r' % reason)
  2074. self.assertEqual(p.output, b'')
  2075. return p.errput
  2076. def test_stdout(self):
  2077. """
  2078. ProcessProtocol.transport.closeStdout actually closes the pipe.
  2079. """
  2080. d = self.doit(1)
  2081. def _check(errput):
  2082. if _PY3:
  2083. if runtime.platform.isWindows():
  2084. self.assertIn(b"OSError", errput)
  2085. self.assertIn(b"22", errput)
  2086. else:
  2087. self.assertIn(b'BrokenPipeError', errput)
  2088. else:
  2089. self.assertIn(b'OSError', errput)
  2090. if runtime.platform.getType() != 'win32':
  2091. self.assertIn(b'Broken pipe', errput)
  2092. d.addCallback(_check)
  2093. return d
  2094. def test_stderr(self):
  2095. """
  2096. ProcessProtocol.transport.closeStderr actually closes the pipe.
  2097. """
  2098. d = self.doit(2)
  2099. def _check(errput):
  2100. # there should be no stderr open, so nothing for it to
  2101. # write the error to.
  2102. self.assertEqual(errput, b'')
  2103. d.addCallback(_check)
  2104. return d
  2105. skipMessage = "wrong platform or reactor doesn't support IReactorProcess"
  2106. if (runtime.platform.getType() != 'posix') or (not interfaces.IReactorProcess(reactor, None)):
  2107. PosixProcessTests.skip = skipMessage
  2108. PosixProcessPTYTests.skip = skipMessage
  2109. TwoProcessesPosixTests.skip = skipMessage
  2110. FDTests.skip = skipMessage
  2111. if (runtime.platform.getType() != 'win32') or (not interfaces.IReactorProcess(reactor, None)):
  2112. Win32ProcessTests.skip = skipMessage
  2113. TwoProcessesNonPosixTests.skip = skipMessage
  2114. DumbWin32ProcTests.skip = skipMessage
  2115. Win32CreateProcessFlagsTests.skip = skipMessage
  2116. Win32UnicodeEnvironmentTests.skip = skipMessage
  2117. if not interfaces.IReactorProcess(reactor, None):
  2118. ProcessTests.skip = skipMessage
  2119. ClosingPipesTests.skip = skipMessage