_settings.py 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740
  1. # coding=utf-8
  2. #
  3. # This file is part of Hypothesis, which may be found at
  4. # https://github.com/HypothesisWorks/hypothesis-python
  5. #
  6. # Most of this work is copyright (C) 2013-2018 David R. MacIver
  7. # (david@drmaciver.com), but it contains contributions by others. See
  8. # CONTRIBUTING.rst for a full list of people who may hold copyright, and
  9. # consult the git log if you need to determine who owns an individual
  10. # contribution.
  11. #
  12. # This Source Code Form is subject to the terms of the Mozilla Public License,
  13. # v. 2.0. If a copy of the MPL was not distributed with this file, You can
  14. # obtain one at http://mozilla.org/MPL/2.0/.
  15. #
  16. # END HEADER
  17. """A module controlling settings for Hypothesis to use in falsification.
  18. Either an explicit settings object can be used or the default object on
  19. this module can be modified.
  20. """
  21. from __future__ import division, print_function, absolute_import
  22. import os
  23. import inspect
  24. import warnings
  25. import threading
  26. from enum import Enum, IntEnum, unique
  27. import attr
  28. from hypothesis.errors import InvalidArgument, HypothesisDeprecationWarning
  29. from hypothesis.configuration import hypothesis_home_dir
  30. from hypothesis.utils.conventions import UniqueIdentifier, not_set
  31. from hypothesis.internal.validation import try_convert
  32. from hypothesis.utils.dynamicvariables import DynamicVariable
  33. __all__ = [
  34. 'settings',
  35. ]
  36. unlimited = UniqueIdentifier('unlimited')
  37. all_settings = {}
  38. _db_cache = {}
  39. class settingsProperty(object):
  40. def __init__(self, name, show_default):
  41. self.name = name
  42. self.show_default = show_default
  43. def __get__(self, obj, type=None):
  44. if obj is None:
  45. return self
  46. else:
  47. try:
  48. return obj.__dict__[self.name]
  49. except KeyError:
  50. raise AttributeError(self.name)
  51. def __set__(self, obj, value):
  52. obj.__dict__[self.name] = value
  53. def __delete__(self, obj):
  54. raise AttributeError('Cannot delete attribute %s' % (self.name,))
  55. @property
  56. def __doc__(self):
  57. description = all_settings[self.name].description
  58. deprecation_message = all_settings[self.name].deprecation_message
  59. default = repr(getattr(settings.default, self.name)) if \
  60. self.show_default else '(dynamically calculated)'
  61. return '\n\n'.join([description, 'default value: %s' % (default,),
  62. (deprecation_message or '').strip()]).strip()
  63. default_variable = DynamicVariable(None)
  64. class settingsMeta(type):
  65. def __init__(self, *args, **kwargs):
  66. super(settingsMeta, self).__init__(*args, **kwargs)
  67. @property
  68. def default(self):
  69. v = default_variable.value
  70. if v is not None:
  71. return v
  72. if hasattr(settings, '_current_profile'):
  73. settings.load_profile(settings._current_profile)
  74. assert default_variable.value is not None
  75. return default_variable.value
  76. @default.setter
  77. def default(self, value):
  78. raise AttributeError('Cannot assign settings.default')
  79. def _assign_default_internal(self, value):
  80. default_variable.value = value
  81. class settings(settingsMeta('settings', (object,), {})):
  82. """A settings object controls a variety of parameters that are used in
  83. falsification. These may control both the falsification strategy and the
  84. details of the data that is generated.
  85. Default values are picked up from the settings.default object and
  86. changes made there will be picked up in newly created settings.
  87. """
  88. _WHITELISTED_REAL_PROPERTIES = [
  89. '_database', '_construction_complete', 'storage'
  90. ]
  91. __definitions_are_locked = False
  92. _profiles = {}
  93. def __getattr__(self, name):
  94. if name in all_settings:
  95. d = all_settings[name].default
  96. if inspect.isfunction(d):
  97. d = d()
  98. return d
  99. else:
  100. raise AttributeError('settings has no attribute %s' % (name,))
  101. def __init__(
  102. self,
  103. parent=None,
  104. **kwargs
  105. ):
  106. self._construction_complete = False
  107. self._database = kwargs.pop('database', not_set)
  108. database_file = kwargs.get('database_file', not_set)
  109. deprecations = []
  110. defaults = parent or settings.default
  111. if defaults is not None:
  112. for setting in all_settings.values():
  113. if kwargs.get(setting.name, not_set) is not_set:
  114. kwargs[setting.name] = getattr(defaults, setting.name)
  115. else:
  116. if kwargs[setting.name] != setting.future_default:
  117. if setting.deprecation_message is not None:
  118. deprecations.append(setting)
  119. if setting.validator:
  120. kwargs[setting.name] = setting.validator(
  121. kwargs[setting.name])
  122. if self._database is not_set and database_file is not_set:
  123. self._database = defaults.database
  124. for name, value in kwargs.items():
  125. if name not in all_settings:
  126. raise InvalidArgument(
  127. 'Invalid argument %s' % (name,))
  128. setattr(self, name, value)
  129. self.storage = threading.local()
  130. self._construction_complete = True
  131. for d in deprecations:
  132. note_deprecation(d.deprecation_message, self)
  133. def defaults_stack(self):
  134. try:
  135. return self.storage.defaults_stack
  136. except AttributeError:
  137. self.storage.defaults_stack = []
  138. return self.storage.defaults_stack
  139. def __call__(self, test):
  140. test._hypothesis_internal_use_settings = self
  141. return test
  142. @classmethod
  143. def define_setting(
  144. cls, name, description, default, options=None, deprecation=None,
  145. validator=None, show_default=True, future_default=not_set,
  146. deprecation_message=None,
  147. ):
  148. """Add a new setting.
  149. - name is the name of the property that will be used to access the
  150. setting. This must be a valid python identifier.
  151. - description will appear in the property's docstring
  152. - default is the default value. This may be a zero argument
  153. function in which case it is evaluated and its result is stored
  154. the first time it is accessed on any given settings object.
  155. """
  156. if settings.__definitions_are_locked:
  157. from hypothesis.errors import InvalidState
  158. raise InvalidState(
  159. 'settings have been locked and may no longer be defined.'
  160. )
  161. if options is not None:
  162. options = tuple(options)
  163. assert default in options
  164. if future_default is not_set:
  165. future_default = default
  166. all_settings[name] = Setting(
  167. name, description.strip(), default, options, validator,
  168. future_default, deprecation_message,
  169. )
  170. setattr(settings, name, settingsProperty(name, show_default))
  171. @classmethod
  172. def lock_further_definitions(cls):
  173. settings.__definitions_are_locked = True
  174. def __setattr__(self, name, value):
  175. if name in settings._WHITELISTED_REAL_PROPERTIES:
  176. return object.__setattr__(self, name, value)
  177. elif name == 'database':
  178. assert self._construction_complete
  179. raise AttributeError(
  180. 'settings objects are immutable and may not be assigned to'
  181. ' after construction.'
  182. )
  183. elif name in all_settings:
  184. if self._construction_complete:
  185. raise AttributeError(
  186. 'settings objects are immutable and may not be assigned to'
  187. ' after construction.'
  188. )
  189. else:
  190. setting = all_settings[name]
  191. if (
  192. setting.options is not None and
  193. value not in setting.options
  194. ):
  195. raise InvalidArgument(
  196. 'Invalid %s, %r. Valid options: %r' % (
  197. name, value, setting.options
  198. )
  199. )
  200. return object.__setattr__(self, name, value)
  201. else:
  202. raise AttributeError('No such setting %s' % (name,))
  203. def __repr__(self):
  204. bits = []
  205. for name in all_settings:
  206. value = getattr(self, name)
  207. bits.append('%s=%r' % (name, value))
  208. bits.sort()
  209. return 'settings(%s)' % ', '.join(bits)
  210. @property
  211. def database(self):
  212. """An ExampleDatabase instance to use for storage of examples. May be
  213. None.
  214. If this was explicitly set at settings instantiation then that
  215. value will be used (even if it was None). If not and the
  216. database_file setting is not None this will be lazily loaded as
  217. an ExampleDatabase, using that file the first time that this
  218. property is accessed on a particular thread.
  219. """
  220. if self._database is not_set and self.database_file is not None:
  221. from hypothesis.database import ExampleDatabase
  222. if self.database_file not in _db_cache:
  223. _db_cache[self.database_file] = (
  224. ExampleDatabase(self.database_file))
  225. return _db_cache[self.database_file]
  226. if self._database is not_set:
  227. self._database = None
  228. return self._database
  229. def __enter__(self):
  230. default_context_manager = default_variable.with_value(self)
  231. self.defaults_stack().append(default_context_manager)
  232. default_context_manager.__enter__()
  233. return self
  234. def __exit__(self, *args, **kwargs):
  235. default_context_manager = self.defaults_stack().pop()
  236. return default_context_manager.__exit__(*args, **kwargs)
  237. @staticmethod
  238. def register_profile(name, settings):
  239. """registers a collection of values to be used as a settings profile.
  240. These settings can be loaded in by name. Enable different defaults for
  241. different settings.
  242. - settings is a settings object
  243. """
  244. settings._profiles[name] = settings
  245. @staticmethod
  246. def get_profile(name):
  247. """Return the profile with the given name.
  248. - name is a string representing the name of the profile
  249. to load
  250. A InvalidArgument exception will be thrown if the
  251. profile does not exist
  252. """
  253. try:
  254. return settings._profiles[name]
  255. except KeyError:
  256. raise InvalidArgument(
  257. "Profile '{0}' has not been registered".format(
  258. name
  259. )
  260. )
  261. @staticmethod
  262. def load_profile(name):
  263. """Loads in the settings defined in the profile provided If the profile
  264. does not exist an InvalidArgument will be thrown.
  265. Any setting not defined in the profile will be the library
  266. defined default for that setting
  267. """
  268. settings._current_profile = name
  269. settings._assign_default_internal(settings.get_profile(name))
  270. @attr.s()
  271. class Setting(object):
  272. name = attr.ib()
  273. description = attr.ib()
  274. default = attr.ib()
  275. options = attr.ib()
  276. validator = attr.ib()
  277. future_default = attr.ib()
  278. deprecation_message = attr.ib()
  279. settings.define_setting(
  280. 'min_satisfying_examples',
  281. default=5,
  282. description="""
  283. Raise Unsatisfiable for any tests which do not produce at least this many
  284. values that pass all assume() calls and which have not exhaustively covered the
  285. search space.
  286. """
  287. )
  288. settings.define_setting(
  289. 'max_examples',
  290. default=100,
  291. description="""
  292. Once this many satisfying examples have been considered without finding any
  293. counter-example, falsification will terminate.
  294. """
  295. )
  296. settings.define_setting(
  297. 'max_iterations',
  298. default=1000,
  299. description="""
  300. Once this many iterations of the example loop have run, including ones which
  301. failed to satisfy assumptions and ones which produced duplicates, falsification
  302. will terminate.
  303. """
  304. )
  305. settings.define_setting(
  306. 'buffer_size',
  307. default=8 * 1024,
  308. description="""
  309. The size of the underlying data used to generate examples. If you need to
  310. generate really large examples you may want to increase this, but it will make
  311. your tests slower.
  312. """
  313. )
  314. settings.define_setting(
  315. 'max_shrinks',
  316. default=500,
  317. description="""
  318. Once this many successful shrinks have been performed, Hypothesis will assume
  319. something has gone a bit wrong and give up rather than continuing to try to
  320. shrink the example.
  321. """
  322. )
  323. def _validate_timeout(n):
  324. if n is unlimited:
  325. return -1
  326. else:
  327. return n
  328. settings.define_setting(
  329. 'timeout',
  330. default=60,
  331. description="""
  332. Once this many seconds have passed, falsify will terminate even
  333. if it has not found many examples. This is a soft rather than a hard
  334. limit - Hypothesis won't e.g. interrupt execution of the called
  335. function to stop it. If this value is <= 0 then no timeout will be
  336. applied.
  337. """,
  338. deprecation_message="""
  339. The timeout setting is deprecated and will be removed in a future version of
  340. Hypothesis. To get the future behaviour set ``timeout=hypothesis.unlimited``
  341. instead (which will remain valid for a further deprecation period after this
  342. setting has gone away).
  343. """,
  344. future_default=unlimited,
  345. validator=_validate_timeout
  346. )
  347. settings.define_setting(
  348. 'derandomize',
  349. default=False,
  350. description="""
  351. If this is True then hypothesis will run in deterministic mode
  352. where each falsification uses a random number generator that is seeded
  353. based on the hypothesis to falsify, which will be consistent across
  354. multiple runs. This has the advantage that it will eliminate any
  355. randomness from your tests, which may be preferable for some situations.
  356. It does have the disadvantage of making your tests less likely to
  357. find novel breakages.
  358. """
  359. )
  360. settings.define_setting(
  361. 'strict',
  362. default=os.getenv('HYPOTHESIS_STRICT_MODE') == 'true',
  363. description="""
  364. If set to True, anything that would cause Hypothesis to issue a warning will
  365. instead raise an error. Note that new warnings may be added at any time, so
  366. running with strict set to True means that new Hypothesis releases may validly
  367. break your code. Note also that, as strict mode is itself deprecated,
  368. enabling it is now an error!
  369. You can enable this setting temporarily by setting the HYPOTHESIS_STRICT_MODE
  370. environment variable to the string 'true'.
  371. """,
  372. deprecation_message="""
  373. Strict mode is deprecated and will go away in a future version of Hypothesis.
  374. To get the same behaviour, use
  375. warnings.simplefilter('error', HypothesisDeprecationWarning).
  376. """,
  377. future_default=False,
  378. )
  379. settings.define_setting(
  380. 'database_file',
  381. default=lambda: (
  382. os.getenv('HYPOTHESIS_DATABASE_FILE') or
  383. os.path.join(hypothesis_home_dir(), 'examples')
  384. ),
  385. show_default=False,
  386. description="""
  387. An instance of hypothesis.database.ExampleDatabase that will be
  388. used to save examples to and load previous examples from. May be None
  389. in which case no storage will be used.
  390. """
  391. )
  392. @unique
  393. class Phase(IntEnum):
  394. explicit = 0
  395. reuse = 1
  396. generate = 2
  397. shrink = 3
  398. @unique
  399. class HealthCheck(Enum):
  400. """Arguments for :attr:`~hypothesis.settings.suppress_health_check`.
  401. Each member of this enum is a type of health check to suppress.
  402. """
  403. exception_in_generation = 0
  404. """Deprecated and no longer does anything. It used to convert errors in
  405. data generation into FailedHealthCheck error."""
  406. data_too_large = 1
  407. """Check for when the typical size of the examples you are generating
  408. exceeds the maximum allowed size too often."""
  409. filter_too_much = 2
  410. """Check for when the test is filtering out too many examples, either
  411. through use of :func:`~hypothesis.assume()` or :ref:`filter() <filtering>`,
  412. or occasionally for Hypothesis internal reasons."""
  413. too_slow = 3
  414. """Check for when your data generation is extremely slow and likely to hurt
  415. testing."""
  416. random_module = 4
  417. """Deprecated and no longer does anything. It used to check for whether
  418. your tests used the global random module. Now @given tests automatically
  419. seed random so this is no longer an error."""
  420. return_value = 5
  421. """Checks if your tests return a non-None value (which will be ignored and
  422. is unlikely to do what you want)."""
  423. hung_test = 6
  424. """Checks if your tests have been running for a very long time."""
  425. large_base_example = 7
  426. """Checks if the natural example to shrink towards is very large."""
  427. @unique
  428. class Statistics(IntEnum):
  429. never = 0
  430. interesting = 1
  431. always = 2
  432. class Verbosity(object):
  433. def __repr__(self):
  434. return 'Verbosity.%s' % (self.name,)
  435. def __init__(self, name, level):
  436. self.name = name
  437. self.level = level
  438. def __eq__(self, other):
  439. return isinstance(other, Verbosity) and (
  440. self.level == other.level
  441. )
  442. def __ne__(self, other):
  443. return not self.__eq__(other)
  444. def __hash__(self):
  445. return self.level
  446. def __lt__(self, other):
  447. return self.level < other.level
  448. def __le__(self, other):
  449. return self.level <= other.level
  450. def __gt__(self, other):
  451. return self.level > other.level
  452. def __ge__(self, other):
  453. return self.level >= other.level
  454. @classmethod
  455. def by_name(cls, key):
  456. result = getattr(cls, key, None)
  457. if isinstance(result, Verbosity):
  458. return result
  459. raise InvalidArgument('No such verbosity level %r' % (key,))
  460. Verbosity.quiet = Verbosity('quiet', 0)
  461. Verbosity.normal = Verbosity('normal', 1)
  462. Verbosity.verbose = Verbosity('verbose', 2)
  463. Verbosity.debug = Verbosity('debug', 3)
  464. Verbosity.all = [
  465. Verbosity.quiet, Verbosity.normal, Verbosity.verbose, Verbosity.debug
  466. ]
  467. ENVIRONMENT_VERBOSITY_OVERRIDE = os.getenv('HYPOTHESIS_VERBOSITY_LEVEL')
  468. if ENVIRONMENT_VERBOSITY_OVERRIDE: # pragma: no cover
  469. DEFAULT_VERBOSITY = Verbosity.by_name(ENVIRONMENT_VERBOSITY_OVERRIDE)
  470. else:
  471. DEFAULT_VERBOSITY = Verbosity.normal
  472. settings.define_setting(
  473. 'verbosity',
  474. options=Verbosity.all,
  475. default=DEFAULT_VERBOSITY,
  476. description='Control the verbosity level of Hypothesis messages',
  477. )
  478. def _validate_phases(phases):
  479. if phases is None:
  480. return tuple(Phase)
  481. phases = tuple(phases)
  482. for a in phases:
  483. if not isinstance(a, Phase):
  484. raise InvalidArgument('%r is not a valid phase' % (a,))
  485. return phases
  486. settings.define_setting(
  487. 'phases',
  488. default=tuple(Phase),
  489. description=(
  490. 'Control which phases should be run. ' +
  491. 'See :ref:`the full documentation for more details <phases>`'
  492. ),
  493. validator=_validate_phases,
  494. )
  495. settings.define_setting(
  496. name='stateful_step_count',
  497. default=50,
  498. description="""
  499. Number of steps to run a stateful program for before giving up on it breaking.
  500. """
  501. )
  502. settings.define_setting(
  503. 'perform_health_check',
  504. default=True,
  505. description=u"""
  506. If set to True, Hypothesis will run a preliminary health check before
  507. attempting to actually execute your test.
  508. """
  509. )
  510. def validate_health_check_suppressions(suppressions):
  511. suppressions = try_convert(list, suppressions, 'suppress_health_check')
  512. for s in suppressions:
  513. if not isinstance(s, HealthCheck):
  514. note_deprecation((
  515. 'Non-HealthCheck value %r of type %s in suppress_health_check '
  516. 'will be ignored, and will become an error in a future '
  517. 'version of Hypothesis') % (
  518. s, type(s).__name__,
  519. ))
  520. elif s in (
  521. HealthCheck.exception_in_generation, HealthCheck.random_module
  522. ):
  523. note_deprecation((
  524. '%s is now ignored and suppressing it is a no-op. This will '
  525. 'become an error in a future version of Hypothesis. Simply '
  526. 'remove it from your list of suppressions to get the same '
  527. 'effect.') % (s,))
  528. return suppressions
  529. settings.define_setting(
  530. 'suppress_health_check',
  531. default=(),
  532. description="""A list of health checks to disable.""",
  533. validator=validate_health_check_suppressions
  534. )
  535. settings.define_setting(
  536. 'deadline',
  537. default=not_set,
  538. description=u"""
  539. If set, a time in milliseconds (which may be a float to express
  540. smaller units of time) that each individual example (i.e. each time your test
  541. function is called, not the whole decorated test) within a test is not
  542. allowed to exceed. Tests which take longer than that may be converted into
  543. errors (but will not necessarily be if close to the deadline, to allow some
  544. variability in test run time).
  545. Set this to None to disable this behaviour entirely.
  546. In future this will default to 200. For now, a
  547. HypothesisDeprecationWarning will be emitted if you exceed that default
  548. deadline and have not explicitly set a deadline yourself.
  549. """
  550. )
  551. settings.define_setting(
  552. 'use_coverage',
  553. default=True,
  554. description="""
  555. Whether to use coverage information to improve Hypothesis's ability to find
  556. bugs.
  557. You should generally leave this turned on unless your code performs
  558. poorly when run under coverage. If you turn it off, please file a bug report
  559. or add a comment to an existing one about the problem that prompted you to do
  560. so.
  561. """
  562. )
  563. class PrintSettings(Enum):
  564. """Flags to determine whether or not to print a detailed example blob to
  565. use with :func:`~hypothesis.reproduce_failure` for failing test cases."""
  566. NEVER = 0
  567. """Never print a blob."""
  568. INFER = 1
  569. """Make an educated guess as to whether it would be appropriate to print
  570. the blob.
  571. The current rules are that this will print if both:
  572. 1. The output from Hypothesis appears to be unsuitable for use with
  573. :func:`~hypothesis.example`.
  574. 2. The output is not too long."""
  575. ALWAYS = 2
  576. """Always print a blob on failure."""
  577. settings.define_setting(
  578. 'print_blob',
  579. default=PrintSettings.INFER,
  580. description="""
  581. Determines whether to print blobs after tests that can be used to reproduce
  582. failures.
  583. See :ref:`the documentation on @reproduce_failure <reproduce_failure>` for
  584. more details of this behaviour.
  585. """
  586. )
  587. settings.lock_further_definitions()
  588. settings.register_profile('default', settings())
  589. settings.load_profile('default')
  590. assert settings.default is not None
  591. def note_deprecation(message, s=None):
  592. if s is None:
  593. s = settings.default
  594. assert s is not None
  595. verbosity = s.verbosity
  596. warning = HypothesisDeprecationWarning(message)
  597. if verbosity > Verbosity.quiet:
  598. warnings.warn(warning, stacklevel=3)