core.py 43 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124
  1. # coding=utf-8
  2. #
  3. # This file is part of Hypothesis, which may be found at
  4. # https://github.com/HypothesisWorks/hypothesis-python
  5. #
  6. # Most of this work is copyright (C) 2013-2018 David R. MacIver
  7. # (david@drmaciver.com), but it contains contributions by others. See
  8. # CONTRIBUTING.rst for a full list of people who may hold copyright, and
  9. # consult the git log if you need to determine who owns an individual
  10. # contribution.
  11. #
  12. # This Source Code Form is subject to the terms of the Mozilla Public License,
  13. # v. 2.0. If a copy of the MPL was not distributed with this file, You can
  14. # obtain one at http://mozilla.org/MPL/2.0/.
  15. #
  16. # END HEADER
  17. """This module provides the core primitives of Hypothesis, such as given."""
  18. from __future__ import division, print_function, absolute_import
  19. import os
  20. import ast
  21. import sys
  22. import time
  23. import zlib
  24. import base64
  25. import traceback
  26. from random import Random
  27. import attr
  28. from coverage import CoverageData
  29. from coverage.files import canonical_filename
  30. from coverage.collector import Collector
  31. import hypothesis.strategies as st
  32. from hypothesis import __version__
  33. from hypothesis.errors import Flaky, Timeout, NoSuchExample, \
  34. Unsatisfiable, DidNotReproduce, InvalidArgument, DeadlineExceeded, \
  35. MultipleFailures, FailedHealthCheck, UnsatisfiedAssumption, \
  36. HypothesisDeprecationWarning
  37. from hypothesis.control import BuildContext
  38. from hypothesis._settings import settings as Settings
  39. from hypothesis._settings import Phase, Verbosity, HealthCheck, \
  40. PrintSettings, note_deprecation
  41. from hypothesis.executors import new_style_executor
  42. from hypothesis.reporting import report, verbose_report, current_verbosity
  43. from hypothesis.statistics import note_engine_for_statistics
  44. from hypothesis.internal.compat import ceil, str_to_bytes, \
  45. benchmark_time, get_type_hints, getfullargspec, encoded_filepath
  46. from hypothesis.internal.coverage import IN_COVERAGE_TESTS
  47. from hypothesis.utils.conventions import infer, not_set
  48. from hypothesis.internal.escalation import is_hypothesis_file, \
  49. escalate_hypothesis_internal_error
  50. from hypothesis.internal.reflection import is_mock, proxies, nicerepr, \
  51. arg_string, impersonate, function_digest, fully_qualified_name, \
  52. define_function_signature, convert_positional_arguments, \
  53. get_pretty_function_description
  54. from hypothesis.internal.healthcheck import fail_health_check
  55. from hypothesis.internal.conjecture.data import StopTest, ConjectureData
  56. from hypothesis.searchstrategy.strategies import SearchStrategy
  57. from hypothesis.internal.conjecture.engine import ExitReason, \
  58. ConjectureRunner, sort_key
  59. try:
  60. from coverage.tracer import CFileDisposition as FileDisposition
  61. except ImportError: # pragma: no cover
  62. from coverage.collector import FileDisposition
  63. running_under_pytest = False
  64. global_force_seed = None
  65. def new_random():
  66. import random
  67. return random.Random(random.getrandbits(128))
  68. @attr.s()
  69. class Example(object):
  70. args = attr.ib()
  71. kwargs = attr.ib()
  72. def example(*args, **kwargs):
  73. """A decorator which ensures a specific example is always tested."""
  74. if args and kwargs:
  75. raise InvalidArgument(
  76. 'Cannot mix positional and keyword arguments for examples'
  77. )
  78. if not (args or kwargs):
  79. raise InvalidArgument(
  80. 'An example must provide at least one argument'
  81. )
  82. def accept(test):
  83. if not hasattr(test, 'hypothesis_explicit_examples'):
  84. test.hypothesis_explicit_examples = []
  85. test.hypothesis_explicit_examples.append(Example(tuple(args), kwargs))
  86. return test
  87. return accept
  88. def seed(seed):
  89. """seed: Start the test execution from a specific seed.
  90. May be any hashable object. No exact meaning for seed is provided
  91. other than that for a fixed seed value Hypothesis will try the same
  92. actions (insofar as it can given external sources of non-
  93. determinism. e.g. timing and hash randomization). Overrides the
  94. derandomize setting if it is present.
  95. """
  96. def accept(test):
  97. test._hypothesis_internal_use_seed = seed
  98. return test
  99. return accept
  100. def reproduce_failure(version, blob):
  101. """Run the example that corresponds to this data blob in order to reproduce
  102. a failure.
  103. A test with this decorator *always* runs only one example and always fails.
  104. If the provided example does not cause a failure, or is in some way invalid
  105. for this test, then this will fail with a DidNotReproduce error.
  106. This decorator is not intended to be a permanent addition to your test
  107. suite. It's simply some code you can add to ease reproduction of a problem
  108. in the event that you don't have access to the test database. Because of
  109. this, *no* compatibility guarantees are made between different versions of
  110. Hypothesis - its API may change arbitrarily from version to version.
  111. """
  112. def accept(test):
  113. test._hypothesis_internal_use_reproduce_failure = (version, blob)
  114. return test
  115. return accept
  116. def encode_failure(buffer):
  117. buffer = bytes(buffer)
  118. compressed = zlib.compress(buffer)
  119. if len(compressed) < len(buffer):
  120. buffer = b'\1' + compressed
  121. else:
  122. buffer = b'\0' + buffer
  123. return base64.b64encode(buffer)
  124. def decode_failure(blob):
  125. try:
  126. buffer = base64.b64decode(blob)
  127. except Exception:
  128. raise InvalidArgument('Invalid base64 encoded string: %r' % (blob,))
  129. prefix = buffer[:1]
  130. if prefix == b'\0':
  131. return buffer[1:]
  132. elif prefix == b'\1':
  133. return zlib.decompress(buffer[1:])
  134. else:
  135. raise InvalidArgument(
  136. 'Could not decode blob %r: Invalid start byte %r' % (
  137. blob, prefix
  138. ))
  139. class WithRunner(SearchStrategy):
  140. def __init__(self, base, runner):
  141. assert runner is not None
  142. self.base = base
  143. self.runner = runner
  144. def do_draw(self, data):
  145. data.hypothesis_runner = self.runner
  146. return self.base.do_draw(data)
  147. def is_invalid_test(
  148. name, original_argspec, generator_arguments, generator_kwargs
  149. ):
  150. def invalid(message):
  151. def wrapped_test(*arguments, **kwargs):
  152. raise InvalidArgument(message)
  153. return wrapped_test
  154. if not (generator_arguments or generator_kwargs):
  155. return invalid(
  156. 'given must be called with at least one argument')
  157. if generator_arguments and any([original_argspec.varargs,
  158. original_argspec.varkw,
  159. original_argspec.kwonlyargs]):
  160. return invalid(
  161. 'positional arguments to @given are not supported with varargs, '
  162. 'varkeywords, or keyword-only arguments'
  163. )
  164. if len(generator_arguments) > len(original_argspec.args):
  165. return invalid((
  166. 'Too many positional arguments for %s() (got %d but'
  167. ' expected at most %d') % (
  168. name, len(generator_arguments),
  169. len(original_argspec.args)))
  170. if infer in generator_arguments:
  171. return invalid('infer was passed as a positional argument to @given, '
  172. 'but may only be passed as a keyword argument')
  173. if generator_arguments and generator_kwargs:
  174. return invalid(
  175. 'cannot mix positional and keyword arguments to @given'
  176. )
  177. extra_kwargs = [
  178. k for k in generator_kwargs
  179. if k not in original_argspec.args + original_argspec.kwonlyargs
  180. ]
  181. if extra_kwargs and not original_argspec.varkw:
  182. return invalid(
  183. '%s() got an unexpected keyword argument %r' % (
  184. name,
  185. extra_kwargs[0]
  186. ))
  187. for a in original_argspec.args:
  188. if isinstance(a, list): # pragma: no cover
  189. return invalid((
  190. 'Cannot decorate function %s() because it has '
  191. 'destructuring arguments') % (
  192. name,
  193. ))
  194. if original_argspec.defaults or original_argspec.kwonlydefaults:
  195. return invalid('Cannot apply @given to a function with defaults.')
  196. missing = [repr(kw) for kw in original_argspec.kwonlyargs
  197. if kw not in generator_kwargs]
  198. if missing:
  199. raise InvalidArgument('Missing required kwarg{}: {}'.format(
  200. 's' if len(missing) > 1 else '', ', '.join(missing)))
  201. def execute_explicit_examples(
  202. test_runner, test, wrapped_test, settings, arguments, kwargs
  203. ):
  204. original_argspec = getfullargspec(test)
  205. for example in reversed(getattr(
  206. wrapped_test, 'hypothesis_explicit_examples', ()
  207. )):
  208. example_kwargs = dict(original_argspec.kwonlydefaults or {})
  209. if example.args:
  210. if len(example.args) > len(original_argspec.args):
  211. raise InvalidArgument(
  212. 'example has too many arguments for test. '
  213. 'Expected at most %d but got %d' % (
  214. len(original_argspec.args), len(example.args)))
  215. example_kwargs.update(dict(zip(
  216. original_argspec.args[-len(example.args):],
  217. example.args
  218. )))
  219. else:
  220. example_kwargs.update(example.kwargs)
  221. if Phase.explicit not in settings.phases:
  222. continue
  223. example_kwargs.update(kwargs)
  224. # Note: Test may mutate arguments and we can't rerun explicit
  225. # examples, so we have to calculate the failure message at this
  226. # point rather than than later.
  227. example_string = '%s(%s)' % (
  228. test.__name__, arg_string(test, arguments, example_kwargs)
  229. )
  230. try:
  231. with BuildContext(None) as b:
  232. if settings.verbosity >= Verbosity.verbose:
  233. report('Trying example: ' + example_string)
  234. test_runner(
  235. None,
  236. lambda data: test(*arguments, **example_kwargs)
  237. )
  238. except BaseException:
  239. report('Falsifying example: ' + example_string)
  240. for n in b.notes:
  241. report(n)
  242. raise
  243. def get_random_for_wrapped_test(test, wrapped_test):
  244. settings = wrapped_test._hypothesis_internal_use_settings
  245. wrapped_test._hypothesis_internal_use_generated_seed = None
  246. if wrapped_test._hypothesis_internal_use_seed is not None:
  247. return Random(
  248. wrapped_test._hypothesis_internal_use_seed)
  249. elif settings.derandomize:
  250. return Random(function_digest(test))
  251. elif global_force_seed is not None:
  252. return Random(global_force_seed)
  253. else:
  254. import random
  255. seed = random.getrandbits(128)
  256. wrapped_test._hypothesis_internal_use_generated_seed = seed
  257. return Random(seed)
  258. def process_arguments_to_given(
  259. wrapped_test, arguments, kwargs, generator_arguments, generator_kwargs,
  260. argspec, test, settings
  261. ):
  262. selfy = None
  263. arguments, kwargs = convert_positional_arguments(
  264. wrapped_test, arguments, kwargs)
  265. # If the test function is a method of some kind, the bound object
  266. # will be the first named argument if there are any, otherwise the
  267. # first vararg (if any).
  268. if argspec.args:
  269. selfy = kwargs.get(argspec.args[0])
  270. elif arguments:
  271. selfy = arguments[0]
  272. # Ensure that we don't mistake mocks for self here.
  273. # This can cause the mock to be used as the test runner.
  274. if is_mock(selfy):
  275. selfy = None
  276. test_runner = new_style_executor(selfy)
  277. arguments = tuple(arguments)
  278. search_strategy = st.tuples(
  279. st.just(arguments),
  280. st.fixed_dictionaries(generator_kwargs).map(
  281. lambda args: dict(args, **kwargs)
  282. )
  283. )
  284. if selfy is not None:
  285. search_strategy = WithRunner(search_strategy, selfy)
  286. search_strategy.validate()
  287. return arguments, kwargs, test_runner, search_strategy
  288. def skip_exceptions_to_reraise():
  289. """Return a tuple of exceptions meaning 'skip this test', to re-raise.
  290. This is intended to cover most common test runners; if you would
  291. like another to be added please open an issue or pull request.
  292. """
  293. import unittest
  294. # This is a set because nose may simply re-export unittest.SkipTest
  295. exceptions = set([unittest.SkipTest])
  296. try: # pragma: no cover
  297. from unittest2 import SkipTest
  298. exceptions.add(SkipTest)
  299. except ImportError:
  300. pass
  301. try: # pragma: no cover
  302. from pytest.runner import Skipped
  303. exceptions.add(Skipped)
  304. except ImportError:
  305. pass
  306. try: # pragma: no cover
  307. from nose import SkipTest as NoseSkipTest
  308. exceptions.add(NoseSkipTest)
  309. except ImportError:
  310. pass
  311. return tuple(sorted(exceptions, key=str))
  312. exceptions_to_reraise = skip_exceptions_to_reraise()
  313. def new_given_argspec(original_argspec, generator_kwargs):
  314. """Make an updated argspec for the wrapped test."""
  315. new_args = [a for a in original_argspec.args if a not in generator_kwargs]
  316. new_kwonlyargs = [a for a in original_argspec.kwonlyargs
  317. if a not in generator_kwargs]
  318. annots = {k: v for k, v in original_argspec.annotations.items()
  319. if k in new_args + new_kwonlyargs}
  320. annots['return'] = None
  321. return original_argspec._replace(
  322. args=new_args, kwonlyargs=new_kwonlyargs, annotations=annots)
  323. ROOT = os.path.dirname(__file__)
  324. STDLIB = os.path.dirname(os.__file__)
  325. def hypothesis_check_include(filename): # pragma: no cover
  326. if is_hypothesis_file(filename):
  327. return False
  328. return filename.endswith('.py')
  329. def escalate_warning(msg, slug=None): # pragma: no cover
  330. if slug is not None:
  331. msg = '%s (%s)' % (msg, slug)
  332. raise AssertionError(
  333. 'Unexpected warning from coverage: %s' % (msg,)
  334. )
  335. class Arc(object):
  336. __slots__ = ('filename', 'source', 'target')
  337. def __init__(self, filename, source, target):
  338. self.filename = filename
  339. self.source = source
  340. self.target = target
  341. ARC_CACHE = {}
  342. def arc(filename, source, target):
  343. try:
  344. return ARC_CACHE[filename][source][target]
  345. except KeyError:
  346. result = Arc(filename, source, target)
  347. ARC_CACHE.setdefault(
  348. filename, {}).setdefault(source, {})[target] = result
  349. return result
  350. in_given = False
  351. FORCE_PURE_TRACER = os.getenv('HYPOTHESIS_FORCE_PURE_TRACER') == 'true'
  352. class StateForActualGivenExecution(object):
  353. def __init__(
  354. self, test_runner, search_strategy, test, settings, random, had_seed
  355. ):
  356. self.test_runner = test_runner
  357. self.search_strategy = search_strategy
  358. self.settings = settings
  359. self.at_least_one_success = False
  360. self.last_exception = None
  361. self.falsifying_examples = ()
  362. self.__was_flaky = False
  363. self.random = random
  364. self.__warned_deadline = False
  365. self.__existing_collector = None
  366. self.__test_runtime = None
  367. self.__had_seed = had_seed
  368. self.test = test
  369. self.coverage_data = CoverageData()
  370. self.files_to_propagate = set()
  371. self.failed_normally = False
  372. self.used_examples_from_database = False
  373. if settings.use_coverage and not IN_COVERAGE_TESTS: # pragma: no cover
  374. if Collector._collectors:
  375. parent = Collector._collectors[-1]
  376. # We include any files the collector has already decided to
  377. # trace whether or not on re-investigation we still think it
  378. # wants to trace them. The reason for this is that in some
  379. # cases coverage gets the wrong answer when we run it
  380. # ourselves due to reasons that are our fault but are hard to
  381. # fix (we lie about where certain functions come from).
  382. # This causes us to not record the actual test bodies as
  383. # covered. But if we intended to trace test bodies then the
  384. # file must already have been traced when getting to this point
  385. # and so will already be in the collector's data. Hence we can
  386. # use that information to get the correct answer here.
  387. # See issue 997 for more context.
  388. self.files_to_propagate = set(parent.data)
  389. self.hijack_collector(parent)
  390. self.collector = Collector(
  391. branch=True,
  392. timid=FORCE_PURE_TRACER,
  393. should_trace=self.should_trace,
  394. check_include=hypothesis_check_include,
  395. concurrency='thread',
  396. warn=escalate_warning,
  397. )
  398. self.collector.reset()
  399. # Hide the other collectors from this one so it doesn't attempt to
  400. # pause them (we're doing trace function management ourselves so
  401. # this will just cause problems).
  402. self.collector._collectors = []
  403. else:
  404. self.collector = None
  405. def execute(
  406. self, data,
  407. print_example=False,
  408. is_final=False,
  409. expected_failure=None, collect=False,
  410. ):
  411. text_repr = [None]
  412. if self.settings.deadline is None:
  413. test = self.test
  414. else:
  415. @proxies(self.test)
  416. def test(*args, **kwargs):
  417. self.__test_runtime = None
  418. initial_draws = len(data.draw_times)
  419. start = benchmark_time()
  420. result = self.test(*args, **kwargs)
  421. finish = benchmark_time()
  422. internal_draw_time = sum(data.draw_times[initial_draws:])
  423. runtime = (finish - start - internal_draw_time) * 1000
  424. self.__test_runtime = runtime
  425. if self.settings.deadline is not_set:
  426. if (
  427. not self.__warned_deadline and
  428. runtime >= 200
  429. ):
  430. self.__warned_deadline = True
  431. note_deprecation((
  432. 'Test took %.2fms to run. In future the default '
  433. 'deadline setting will be 200ms, which will '
  434. 'make this an error. You can set deadline to '
  435. 'an explicit value of e.g. %d to turn tests '
  436. 'slower than this into an error, or you can set '
  437. 'it to None to disable this check entirely.') % (
  438. runtime, ceil(runtime / 100) * 100,
  439. ))
  440. else:
  441. current_deadline = self.settings.deadline
  442. if not is_final:
  443. current_deadline *= 1.25
  444. if runtime >= current_deadline:
  445. raise DeadlineExceeded(runtime, self.settings.deadline)
  446. return result
  447. def run(data):
  448. if not hasattr(data, 'can_reproduce_example_from_repr'):
  449. data.can_reproduce_example_from_repr = True
  450. with self.settings:
  451. with BuildContext(data, is_final=is_final):
  452. import random as rnd_module
  453. rnd_module.seed(0)
  454. args, kwargs = data.draw(self.search_strategy)
  455. if expected_failure is not None:
  456. text_repr[0] = arg_string(test, args, kwargs)
  457. if print_example:
  458. example = '%s(%s)' % (
  459. test.__name__, arg_string(test, args, kwargs))
  460. try:
  461. ast.parse(example)
  462. except SyntaxError:
  463. data.can_reproduce_example_from_repr = False
  464. report('Falsifying example: %s' % (example,))
  465. elif current_verbosity() >= Verbosity.verbose:
  466. report(
  467. lambda: 'Trying example: %s(%s)' % (
  468. test.__name__, arg_string(test, args, kwargs)))
  469. if self.collector is None or not collect:
  470. return test(*args, **kwargs)
  471. else: # pragma: no cover
  472. try:
  473. self.collector.start()
  474. return test(*args, **kwargs)
  475. finally:
  476. self.collector.stop()
  477. result = self.test_runner(data, run)
  478. if expected_failure is not None:
  479. exception, traceback = expected_failure
  480. if (
  481. isinstance(
  482. exception,
  483. DeadlineExceeded
  484. ) and self.__test_runtime is not None
  485. ):
  486. report((
  487. 'Unreliable test timings! On an initial run, this '
  488. 'test took %.2fms, which exceeded the deadline of '
  489. '%.2fms, but on a subsequent run it took %.2f ms, '
  490. 'which did not. If you expect this sort of '
  491. 'variability in your test timings, consider turning '
  492. 'deadlines off for this test by setting deadline=None.'
  493. ) % (
  494. exception.runtime,
  495. self.settings.deadline, self.__test_runtime
  496. ))
  497. else:
  498. report(
  499. 'Failed to reproduce exception. Expected: \n' +
  500. traceback,
  501. )
  502. self.__flaky((
  503. 'Hypothesis %s(%s) produces unreliable results: Falsified'
  504. ' on the first call but did not on a subsequent one'
  505. ) % (test.__name__, text_repr[0],))
  506. return result
  507. def should_trace(self, original_filename, frame): # pragma: no cover
  508. disp = FileDisposition()
  509. assert original_filename is not None
  510. disp.original_filename = original_filename
  511. disp.canonical_filename = encoded_filepath(
  512. canonical_filename(original_filename))
  513. disp.source_filename = disp.canonical_filename
  514. disp.reason = ''
  515. disp.file_tracer = None
  516. disp.has_dynamic_filename = False
  517. disp.trace = hypothesis_check_include(disp.canonical_filename)
  518. if not disp.trace:
  519. disp.reason = 'hypothesis internal reasons'
  520. elif self.__existing_collector is not None:
  521. check = self.__existing_collector.should_trace(
  522. original_filename, frame)
  523. if check.trace:
  524. self.files_to_propagate.add(check.canonical_filename)
  525. return disp
  526. def hijack_collector(self, collector): # pragma: no cover
  527. self.__existing_collector = collector
  528. original_save_data = collector.save_data
  529. def save_data(covdata):
  530. original_save_data(covdata)
  531. if collector.branch:
  532. covdata.add_arcs({
  533. filename: {
  534. arc: None
  535. for arc in self.coverage_data.arcs(filename) or ()}
  536. for filename in self.files_to_propagate
  537. })
  538. else:
  539. covdata.add_lines({
  540. filename: {
  541. line: None
  542. for line in self.coverage_data.lines(filename) or ()}
  543. for filename in self.files_to_propagate
  544. })
  545. collector.save_data = original_save_data
  546. collector.save_data = save_data
  547. def evaluate_test_data(self, data):
  548. try:
  549. if self.collector is None:
  550. result = self.execute(data)
  551. else: # pragma: no cover
  552. # This should always be a no-op, but the coverage tracer has
  553. # a bad habit of resurrecting itself.
  554. original = sys.gettrace()
  555. sys.settrace(None)
  556. try:
  557. self.collector.data = {}
  558. result = self.execute(data, collect=True)
  559. finally:
  560. sys.settrace(original)
  561. covdata = CoverageData()
  562. self.collector.save_data(covdata)
  563. self.coverage_data.update(covdata)
  564. for filename in covdata.measured_files():
  565. if is_hypothesis_file(filename):
  566. continue
  567. data.tags.update(
  568. arc(filename, source, target)
  569. for source, target in covdata.arcs(filename)
  570. )
  571. if result is not None and self.settings.perform_health_check:
  572. fail_health_check(self.settings, (
  573. 'Tests run under @given should return None, but '
  574. '%s returned %r instead.'
  575. ) % (self.test.__name__, result), HealthCheck.return_value)
  576. self.at_least_one_success = True
  577. return False
  578. except UnsatisfiedAssumption:
  579. data.mark_invalid()
  580. except (
  581. HypothesisDeprecationWarning, FailedHealthCheck,
  582. StopTest,
  583. ) + exceptions_to_reraise:
  584. raise
  585. except Exception as e:
  586. escalate_hypothesis_internal_error()
  587. data.__expected_traceback = traceback.format_exc()
  588. data.__expected_exception = e
  589. verbose_report(data.__expected_traceback)
  590. error_class, _, tb = sys.exc_info()
  591. origin = traceback.extract_tb(tb)[-1]
  592. filename = origin[0]
  593. lineno = origin[1]
  594. data.mark_interesting((error_class, filename, lineno))
  595. def run(self):
  596. # Tell pytest to omit the body of this function from tracebacks
  597. __tracebackhide__ = True
  598. if global_force_seed is None:
  599. database_key = str_to_bytes(fully_qualified_name(self.test))
  600. else:
  601. database_key = None
  602. self.start_time = time.time()
  603. global in_given
  604. runner = ConjectureRunner(
  605. self.evaluate_test_data,
  606. settings=self.settings, random=self.random,
  607. database_key=database_key,
  608. )
  609. if in_given or self.collector is None:
  610. runner.run()
  611. else: # pragma: no cover
  612. in_given = True
  613. original_trace = sys.gettrace()
  614. try:
  615. sys.settrace(None)
  616. runner.run()
  617. finally:
  618. in_given = False
  619. sys.settrace(original_trace)
  620. self.used_examples_from_database = \
  621. runner.used_examples_from_database
  622. note_engine_for_statistics(runner)
  623. run_time = time.time() - self.start_time
  624. self.used_examples_from_database = runner.used_examples_from_database
  625. if runner.used_examples_from_database:
  626. if self.settings.derandomize:
  627. note_deprecation(
  628. 'In future derandomize will imply database=None, but your '
  629. 'test is currently using examples from the database. To '
  630. 'get the future behaviour, update your settings to '
  631. 'include database=None.'
  632. )
  633. if self.__had_seed:
  634. note_deprecation(
  635. 'In future use of @seed will imply database=None in your '
  636. 'settings, but your test is currently using examples from '
  637. 'the database. To get the future behaviour, update your '
  638. 'settings for this test to include database=None.'
  639. )
  640. timed_out = runner.exit_reason == ExitReason.timeout
  641. if runner.call_count == 0:
  642. return
  643. if runner.interesting_examples:
  644. self.falsifying_examples = sorted(
  645. [d for d in runner.interesting_examples.values()],
  646. key=lambda d: sort_key(d.buffer), reverse=True
  647. )
  648. else:
  649. if timed_out:
  650. note_deprecation((
  651. 'Your tests are hitting the settings timeout (%.2fs). '
  652. 'This functionality will go away in a future release '
  653. 'and you should not rely on it. Instead, try setting '
  654. 'max_examples to be some value lower than %d (the number '
  655. 'of examples your test successfully ran here). Or, if you '
  656. 'would prefer your tests to run to completion, regardless '
  657. 'of how long they take, you can set the timeout value to '
  658. 'hypothesis.unlimited.'
  659. ) % (
  660. self.settings.timeout, runner.valid_examples),
  661. self.settings)
  662. if runner.valid_examples < min(
  663. self.settings.min_satisfying_examples,
  664. self.settings.max_examples,
  665. ) and not (
  666. runner.exit_reason == ExitReason.finished and
  667. self.at_least_one_success
  668. ):
  669. if timed_out:
  670. raise Timeout((
  671. 'Ran out of time before finding a satisfying '
  672. 'example for '
  673. '%s. Only found %d examples in ' +
  674. '%.2fs.'
  675. ) % (
  676. get_pretty_function_description(self.test),
  677. runner.valid_examples, run_time
  678. ))
  679. else:
  680. raise Unsatisfiable((
  681. 'Unable to satisfy assumptions of hypothesis '
  682. '%s. Only %d examples considered '
  683. 'satisfied assumptions'
  684. ) % (
  685. get_pretty_function_description(self.test),
  686. runner.valid_examples,))
  687. if not self.falsifying_examples:
  688. return
  689. self.failed_normally = True
  690. flaky = 0
  691. for falsifying_example in self.falsifying_examples:
  692. ran_example = ConjectureData.for_buffer(falsifying_example.buffer)
  693. self.__was_flaky = False
  694. assert falsifying_example.__expected_exception is not None
  695. try:
  696. self.execute(
  697. ran_example,
  698. print_example=True, is_final=True,
  699. expected_failure=(
  700. falsifying_example.__expected_exception,
  701. falsifying_example.__expected_traceback,
  702. )
  703. )
  704. except (UnsatisfiedAssumption, StopTest):
  705. report(traceback.format_exc())
  706. self.__flaky(
  707. 'Unreliable assumption: An example which satisfied '
  708. 'assumptions on the first run now fails it.'
  709. )
  710. except BaseException:
  711. if len(self.falsifying_examples) <= 1:
  712. raise
  713. report(traceback.format_exc())
  714. finally: # pragma: no cover
  715. # This section is in fact entirely covered by the tests in
  716. # test_reproduce_failure, but it seems to trigger a lovely set
  717. # of coverage bugs: The branches show up as uncovered (despite
  718. # definitely being covered - you can add an assert False else
  719. # branch to verify this and see it fail - and additionally the
  720. # second branch still complains about lack of coverage even if
  721. # you add a pragma: no cover to it!
  722. # See https://bitbucket.org/ned/coveragepy/issues/623/
  723. if self.settings.print_blob is not PrintSettings.NEVER:
  724. failure_blob = encode_failure(falsifying_example.buffer)
  725. # Have to use the example we actually ran, not the original
  726. # falsifying example! Otherwise we won't catch problems
  727. # where the repr of the generated example doesn't parse.
  728. can_use_repr = ran_example.can_reproduce_example_from_repr
  729. if (
  730. self.settings.print_blob is PrintSettings.ALWAYS or (
  731. self.settings.print_blob is PrintSettings.INFER and
  732. not can_use_repr and
  733. len(failure_blob) < 200
  734. )
  735. ):
  736. report((
  737. '\n'
  738. 'You can reproduce this example by temporarily '
  739. 'adding @reproduce_failure(%r, %r) as a decorator '
  740. 'on your test case') % (
  741. __version__, failure_blob,))
  742. if self.__was_flaky:
  743. flaky += 1
  744. # If we only have one example then we should have raised an error or
  745. # flaky prior to this point.
  746. assert len(self.falsifying_examples) > 1
  747. if flaky > 0:
  748. raise Flaky((
  749. 'Hypothesis found %d distinct failures, but %d of them '
  750. 'exhibited some sort of flaky behaviour.') % (
  751. len(self.falsifying_examples), flaky))
  752. else:
  753. raise MultipleFailures((
  754. 'Hypothesis found %d distinct failures.') % (
  755. len(self.falsifying_examples,)))
  756. def __flaky(self, message):
  757. if len(self.falsifying_examples) <= 1:
  758. raise Flaky(message)
  759. else:
  760. self.__was_flaky = True
  761. report('Flaky example! ' + message)
  762. def given(*given_arguments, **given_kwargs):
  763. """A decorator for turning a test function that accepts arguments into a
  764. randomized test.
  765. This is the main entry point to Hypothesis.
  766. """
  767. def run_test_with_generator(test):
  768. generator_arguments = tuple(given_arguments)
  769. generator_kwargs = dict(given_kwargs)
  770. original_argspec = getfullargspec(test)
  771. check_invalid = is_invalid_test(
  772. test.__name__, original_argspec,
  773. generator_arguments, generator_kwargs)
  774. if check_invalid is not None:
  775. return check_invalid
  776. for name, strategy in zip(reversed(original_argspec.args),
  777. reversed(generator_arguments)):
  778. generator_kwargs[name] = strategy
  779. argspec = new_given_argspec(original_argspec, generator_kwargs)
  780. @impersonate(test)
  781. @define_function_signature(
  782. test.__name__, test.__doc__, argspec
  783. )
  784. def wrapped_test(*arguments, **kwargs):
  785. # Tell pytest to omit the body of this function from tracebacks
  786. __tracebackhide__ = True
  787. if getattr(test, 'is_hypothesis_test', False):
  788. note_deprecation((
  789. 'You have applied @given to a test more than once. In '
  790. 'future this will be an error. Applying @given twice '
  791. 'wraps the test twice, which can be extremely slow. A '
  792. 'similar effect can be gained by combining the arguments '
  793. 'to the two calls to given. For example, instead of '
  794. '@given(booleans()) @given(integers()), you could write '
  795. '@given(booleans(), integers())'
  796. ))
  797. settings = wrapped_test._hypothesis_internal_use_settings
  798. random = get_random_for_wrapped_test(test, wrapped_test)
  799. if infer in generator_kwargs.values():
  800. hints = get_type_hints(test)
  801. for name in [name for name, value in generator_kwargs.items()
  802. if value is infer]:
  803. if name not in hints:
  804. raise InvalidArgument(
  805. 'passed %s=infer for %s, but %s has no type annotation'
  806. % (name, test.__name__, name))
  807. generator_kwargs[name] = st.from_type(hints[name])
  808. processed_args = process_arguments_to_given(
  809. wrapped_test, arguments, kwargs, generator_arguments,
  810. generator_kwargs, argspec, test, settings
  811. )
  812. arguments, kwargs, test_runner, search_strategy = processed_args
  813. state = StateForActualGivenExecution(
  814. test_runner, search_strategy, test, settings, random,
  815. had_seed=wrapped_test._hypothesis_internal_use_seed
  816. )
  817. reproduce_failure = \
  818. wrapped_test._hypothesis_internal_use_reproduce_failure
  819. if reproduce_failure is not None:
  820. expected_version, failure = reproduce_failure
  821. if expected_version != __version__:
  822. raise InvalidArgument((
  823. 'Attempting to reproduce a failure from a different '
  824. 'version of Hypothesis. This failure is from %s, but '
  825. 'you are currently running %r. Please change your '
  826. 'Hypothesis version to a matching one.'
  827. ) % (expected_version, __version__))
  828. try:
  829. state.execute(ConjectureData.for_buffer(
  830. decode_failure(failure)),
  831. print_example=True, is_final=True,
  832. )
  833. raise DidNotReproduce(
  834. 'Expected the test to raise an error, but it '
  835. 'completed successfully.'
  836. )
  837. except StopTest:
  838. raise DidNotReproduce(
  839. 'The shape of the test data has changed in some way '
  840. 'from where this blob was defined. Are you sure '
  841. "you're running the same test?"
  842. )
  843. except UnsatisfiedAssumption:
  844. raise DidNotReproduce(
  845. 'The test data failed to satisfy an assumption in the '
  846. 'test. Have you added it since this blob was '
  847. 'generated?'
  848. )
  849. execute_explicit_examples(
  850. test_runner, test, wrapped_test, settings, arguments, kwargs
  851. )
  852. if settings.max_examples <= 0:
  853. return
  854. if not (
  855. Phase.reuse in settings.phases or
  856. Phase.generate in settings.phases
  857. ):
  858. return
  859. try:
  860. state.run()
  861. except BaseException:
  862. generated_seed = \
  863. wrapped_test._hypothesis_internal_use_generated_seed
  864. if generated_seed is not None and not state.failed_normally:
  865. if running_under_pytest:
  866. report((
  867. 'You can add @seed(%(seed)d) to this test or run '
  868. 'pytest with --hypothesis-seed=%(seed)d to '
  869. 'reproduce this failure.') % {
  870. 'seed': generated_seed},)
  871. else:
  872. report((
  873. 'You can add @seed(%d) to this test to reproduce '
  874. 'this failure.') % (generated_seed,))
  875. raise
  876. for attrib in dir(test):
  877. if not (attrib.startswith('_') or hasattr(wrapped_test, attrib)):
  878. setattr(wrapped_test, attrib, getattr(test, attrib))
  879. wrapped_test.is_hypothesis_test = True
  880. wrapped_test._hypothesis_internal_use_seed = getattr(
  881. test, '_hypothesis_internal_use_seed', None
  882. )
  883. wrapped_test._hypothesis_internal_use_settings = getattr(
  884. test, '_hypothesis_internal_use_settings', None
  885. ) or Settings.default
  886. wrapped_test._hypothesis_internal_use_reproduce_failure = getattr(
  887. test, '_hypothesis_internal_use_reproduce_failure', None
  888. )
  889. return wrapped_test
  890. return run_test_with_generator
  891. def find(specifier, condition, settings=None, random=None, database_key=None):
  892. """Returns the minimal example from the given strategy ``specifier`` that
  893. matches the predicate function ``condition``."""
  894. settings = settings or Settings(
  895. max_examples=2000,
  896. min_satisfying_examples=0,
  897. max_shrinks=2000,
  898. )
  899. settings = Settings(settings, perform_health_check=False)
  900. if database_key is None and settings.database is not None:
  901. database_key = function_digest(condition)
  902. if not isinstance(specifier, SearchStrategy):
  903. raise InvalidArgument(
  904. 'Expected SearchStrategy but got %r of type %s' % (
  905. specifier, type(specifier).__name__
  906. ))
  907. specifier.validate()
  908. search = specifier
  909. random = random or new_random()
  910. successful_examples = [0]
  911. last_data = [None]
  912. def template_condition(data):
  913. with BuildContext(data):
  914. try:
  915. data.is_find = True
  916. result = data.draw(search)
  917. data.note(result)
  918. success = condition(result)
  919. except UnsatisfiedAssumption:
  920. data.mark_invalid()
  921. if success:
  922. successful_examples[0] += 1
  923. if settings.verbosity == Verbosity.verbose:
  924. if not successful_examples[0]:
  925. report(lambda: u'Trying example %s' % (
  926. nicerepr(result),
  927. ))
  928. elif success:
  929. if successful_examples[0] == 1:
  930. report(lambda: u'Found satisfying example %s' % (
  931. nicerepr(result),
  932. ))
  933. else:
  934. report(lambda: u'Shrunk example to %s' % (
  935. nicerepr(result),
  936. ))
  937. last_data[0] = data
  938. if success and not data.frozen:
  939. data.mark_interesting()
  940. start = time.time()
  941. runner = ConjectureRunner(
  942. template_condition, settings=settings, random=random,
  943. database_key=database_key,
  944. )
  945. runner.run()
  946. note_engine_for_statistics(runner)
  947. run_time = time.time() - start
  948. if runner.interesting_examples:
  949. data = ConjectureData.for_buffer(
  950. list(runner.interesting_examples.values())[0].buffer)
  951. with BuildContext(data):
  952. return data.draw(search)
  953. if (
  954. runner.valid_examples <= settings.min_satisfying_examples and
  955. runner.exit_reason != ExitReason.finished
  956. ):
  957. if settings.timeout > 0 and run_time > settings.timeout:
  958. raise Timeout((
  959. 'Ran out of time before finding enough valid examples for '
  960. '%s. Only %d valid examples found in %.2f seconds.'
  961. ) % (
  962. get_pretty_function_description(condition),
  963. runner.valid_examples, run_time))
  964. else:
  965. raise Unsatisfiable((
  966. 'Unable to satisfy assumptions of '
  967. '%s. Only %d examples considered satisfied assumptions'
  968. ) % (
  969. get_pretty_function_description(condition),
  970. runner.valid_examples,))
  971. raise NoSuchExample(get_pretty_function_description(condition))