_suite.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. """
  2. Python representations of the JSON Schema Test Suite tests.
  3. """
  4. from functools import partial
  5. import json
  6. import os
  7. import re
  8. import subprocess
  9. import sys
  10. import unittest
  11. from twisted.python.filepath import FilePath
  12. import attr
  13. from jsonschema.compat import PY3
  14. from jsonschema.validators import validators
  15. import jsonschema
  16. def _find_suite():
  17. root = os.environ.get("JSON_SCHEMA_TEST_SUITE")
  18. if root is not None:
  19. return FilePath(root)
  20. root = FilePath(jsonschema.__file__).parent().sibling("json")
  21. if not root.isdir(): # pragma: no cover
  22. raise ValueError(
  23. (
  24. "Can't find the JSON-Schema-Test-Suite directory. "
  25. "Set the 'JSON_SCHEMA_TEST_SUITE' environment "
  26. "variable or run the tests from alongside a checkout "
  27. "of the suite."
  28. ),
  29. )
  30. return root
  31. @attr.s(hash=True)
  32. class Suite(object):
  33. _root = attr.ib(default=attr.Factory(_find_suite))
  34. def _remotes(self):
  35. jsonschema_suite = self._root.descendant(["bin", "jsonschema_suite"])
  36. remotes = subprocess.check_output(
  37. [sys.executable, jsonschema_suite.path, "remotes"],
  38. )
  39. return {
  40. "http://localhost:1234/" + name: schema
  41. for name, schema in json.loads(remotes.decode("utf-8")).items()
  42. }
  43. def benchmark(self, runner): # pragma: no cover
  44. for name in validators:
  45. self.version(name=name).benchmark(runner=runner)
  46. def version(self, name):
  47. return Version(
  48. name=name,
  49. path=self._root.descendant(["tests", name]),
  50. remotes=self._remotes(),
  51. )
  52. @attr.s(hash=True)
  53. class Version(object):
  54. _path = attr.ib()
  55. _remotes = attr.ib()
  56. name = attr.ib()
  57. def benchmark(self, runner, **kwargs): # pragma: no cover
  58. for suite in self.tests():
  59. for test in suite:
  60. runner.bench_func(
  61. test.fully_qualified_name,
  62. partial(test.validate_ignoring_errors, **kwargs),
  63. )
  64. def tests(self):
  65. return (
  66. test
  67. for child in self._path.globChildren("*.json")
  68. for test in self._tests_in(
  69. subject=child.basename()[:-5],
  70. path=child,
  71. )
  72. )
  73. def format_tests(self):
  74. path = self._path.descendant(["optional", "format"])
  75. return (
  76. test
  77. for child in path.globChildren("*.json")
  78. for test in self._tests_in(
  79. subject=child.basename()[:-5],
  80. path=child,
  81. )
  82. )
  83. def tests_of(self, name):
  84. return self._tests_in(
  85. subject=name,
  86. path=self._path.child(name + ".json"),
  87. )
  88. def optional_tests_of(self, name):
  89. return self._tests_in(
  90. subject=name,
  91. path=self._path.descendant(["optional", name + ".json"]),
  92. )
  93. def to_unittest_testcase(self, *suites, **kwargs):
  94. name = kwargs.pop("name", "Test" + self.name.title())
  95. methods = {
  96. test.method_name: test.to_unittest_method(**kwargs)
  97. for suite in suites
  98. for tests in suite
  99. for test in tests
  100. }
  101. cls = type(name, (unittest.TestCase,), methods)
  102. try:
  103. cls.__module__ = _someone_save_us_the_module_of_the_caller()
  104. except Exception: # pragma: no cover
  105. # We're doing crazy things, so if they go wrong, like a function
  106. # behaving differently on some other interpreter, just make them
  107. # not happen.
  108. pass
  109. return cls
  110. def _tests_in(self, subject, path):
  111. for each in json.loads(path.getContent().decode("utf-8")):
  112. yield (
  113. _Test(
  114. version=self,
  115. subject=subject,
  116. case_description=each["description"],
  117. schema=each["schema"],
  118. remotes=self._remotes,
  119. **test
  120. ) for test in each["tests"]
  121. )
  122. @attr.s(hash=True, repr=False)
  123. class _Test(object):
  124. version = attr.ib()
  125. subject = attr.ib()
  126. case_description = attr.ib()
  127. description = attr.ib()
  128. data = attr.ib()
  129. schema = attr.ib(repr=False)
  130. valid = attr.ib()
  131. _remotes = attr.ib()
  132. def __repr__(self): # pragma: no cover
  133. return "<Test {}>".format(self.fully_qualified_name)
  134. @property
  135. def fully_qualified_name(self): # pragma: no cover
  136. return " > ".join(
  137. [
  138. self.version.name,
  139. self.subject,
  140. self.case_description,
  141. self.description,
  142. ]
  143. )
  144. @property
  145. def method_name(self):
  146. delimiters = r"[\W\- ]+"
  147. name = "test_%s_%s_%s" % (
  148. re.sub(delimiters, "_", self.subject),
  149. re.sub(delimiters, "_", self.case_description),
  150. re.sub(delimiters, "_", self.description),
  151. )
  152. if not PY3: # pragma: no cover
  153. name = name.encode("utf-8")
  154. return name
  155. def to_unittest_method(self, skip=lambda test: None, **kwargs):
  156. if self.valid:
  157. def fn(this):
  158. self.validate(**kwargs)
  159. else:
  160. def fn(this):
  161. with this.assertRaises(jsonschema.ValidationError):
  162. self.validate(**kwargs)
  163. fn.__name__ = self.method_name
  164. reason = skip(self)
  165. return unittest.skipIf(reason is not None, reason)(fn)
  166. def validate(self, Validator, **kwargs):
  167. resolver = jsonschema.RefResolver.from_schema(
  168. schema=self.schema,
  169. store=self._remotes,
  170. id_of=Validator.ID_OF,
  171. )
  172. jsonschema.validate(
  173. instance=self.data,
  174. schema=self.schema,
  175. cls=Validator,
  176. resolver=resolver,
  177. **kwargs
  178. )
  179. def validate_ignoring_errors(self, Validator): # pragma: no cover
  180. try:
  181. self.validate(Validator=Validator)
  182. except jsonschema.ValidationError:
  183. pass
  184. def _someone_save_us_the_module_of_the_caller():
  185. """
  186. The FQON of the module 2nd stack frames up from here.
  187. This is intended to allow us to dynamicallly return test case classes that
  188. are indistinguishable from being defined in the module that wants them.
  189. Otherwise, trial will mis-print the FQON, and copy pasting it won't re-run
  190. the class that really is running.
  191. Save us all, this is all so so so so so terrible.
  192. """
  193. return sys._getframe(2).f_globals["__name__"]