test_jsonschema_test_suite.py 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277
  1. """
  2. Test runner for the JSON Schema official test suite
  3. Tests comprehensive correctness of each draft's validator.
  4. See https://github.com/json-schema-org/JSON-Schema-Test-Suite for details.
  5. """
  6. import sys
  7. import warnings
  8. from jsonschema import (
  9. Draft3Validator,
  10. Draft4Validator,
  11. Draft6Validator,
  12. Draft7Validator,
  13. draft3_format_checker,
  14. draft4_format_checker,
  15. draft6_format_checker,
  16. draft7_format_checker,
  17. )
  18. from jsonschema.tests._helpers import bug
  19. from jsonschema.tests._suite import Suite
  20. from jsonschema.validators import _DEPRECATED_DEFAULT_TYPES, create
  21. SUITE = Suite()
  22. DRAFT3 = SUITE.version(name="draft3")
  23. DRAFT4 = SUITE.version(name="draft4")
  24. DRAFT6 = SUITE.version(name="draft6")
  25. DRAFT7 = SUITE.version(name="draft7")
  26. def skip(message, **kwargs):
  27. def skipper(test):
  28. if all(value == getattr(test, attr) for attr, value in kwargs.items()):
  29. return message
  30. return skipper
  31. def missing_format(checker):
  32. def missing_format(test):
  33. schema = test.schema
  34. if schema is True or schema is False or "format" not in schema:
  35. return
  36. if schema["format"] not in checker.checkers:
  37. return "Format checker {0!r} not found.".format(schema["format"])
  38. return missing_format
  39. is_narrow_build = sys.maxunicode == 2 ** 16 - 1
  40. if is_narrow_build: # pragma: no cover
  41. message = "Not running surrogate Unicode case, this Python is narrow."
  42. def narrow_unicode_build(test): # pragma: no cover
  43. return skip(
  44. message=message,
  45. description="one supplementary Unicode code point is not long enough",
  46. )(test) or skip(
  47. message=message,
  48. description="two supplementary Unicode code points is long enough",
  49. )(test)
  50. else:
  51. def narrow_unicode_build(test): # pragma: no cover
  52. return
  53. TestDraft3 = DRAFT3.to_unittest_testcase(
  54. DRAFT3.tests(),
  55. DRAFT3.optional_tests_of(name="bignum"),
  56. DRAFT3.optional_tests_of(name="format"),
  57. DRAFT3.optional_tests_of(name="zeroTerminatedFloats"),
  58. Validator=Draft3Validator,
  59. format_checker=draft3_format_checker,
  60. skip=lambda test: (
  61. narrow_unicode_build(test)
  62. or missing_format(draft3_format_checker)(test)
  63. or skip(
  64. message="Upstream bug in strict_rfc3339",
  65. subject="format",
  66. description="case-insensitive T and Z",
  67. )(test)
  68. ),
  69. )
  70. TestDraft4 = DRAFT4.to_unittest_testcase(
  71. DRAFT4.tests(),
  72. DRAFT4.optional_tests_of(name="bignum"),
  73. DRAFT4.optional_tests_of(name="format"),
  74. DRAFT4.optional_tests_of(name="zeroTerminatedFloats"),
  75. Validator=Draft4Validator,
  76. format_checker=draft4_format_checker,
  77. skip=lambda test: (
  78. narrow_unicode_build(test)
  79. or missing_format(draft4_format_checker)(test)
  80. or skip(
  81. message=bug(),
  82. subject="ref",
  83. case_description="Recursive references between schemas",
  84. )(test)
  85. or skip(
  86. message=bug(371),
  87. subject="ref",
  88. case_description="Location-independent identifier",
  89. )(test)
  90. or skip(
  91. message=bug(371),
  92. subject="ref",
  93. case_description=(
  94. "Location-independent identifier with absolute URI"
  95. ),
  96. )(test)
  97. or skip(
  98. message=bug(371),
  99. subject="ref",
  100. case_description=(
  101. "Location-independent identifier with base URI change in subschema"
  102. ),
  103. )(test)
  104. or skip(
  105. message=bug(),
  106. subject="refRemote",
  107. case_description="base URI change - change folder in subschema",
  108. )(test)
  109. or skip(
  110. message="Upstream bug in strict_rfc3339",
  111. subject="format",
  112. description="case-insensitive T and Z",
  113. )(test)
  114. ),
  115. )
  116. TestDraft6 = DRAFT6.to_unittest_testcase(
  117. DRAFT6.tests(),
  118. DRAFT6.optional_tests_of(name="bignum"),
  119. DRAFT6.optional_tests_of(name="format"),
  120. DRAFT6.optional_tests_of(name="zeroTerminatedFloats"),
  121. Validator=Draft6Validator,
  122. format_checker=draft6_format_checker,
  123. skip=lambda test: (
  124. narrow_unicode_build(test)
  125. or missing_format(draft6_format_checker)(test)
  126. or skip(
  127. message=bug(),
  128. subject="ref",
  129. case_description="Recursive references between schemas",
  130. )(test)
  131. or skip(
  132. message=bug(371),
  133. subject="ref",
  134. case_description="Location-independent identifier",
  135. )(test)
  136. or skip(
  137. message=bug(371),
  138. subject="ref",
  139. case_description=(
  140. "Location-independent identifier with absolute URI"
  141. ),
  142. )(test)
  143. or skip(
  144. message=bug(371),
  145. subject="ref",
  146. case_description=(
  147. "Location-independent identifier with base URI change in subschema"
  148. ),
  149. )(test)
  150. or skip(
  151. message=bug(),
  152. subject="refRemote",
  153. case_description="base URI change - change folder in subschema",
  154. )(test)
  155. or skip(
  156. message="Upstream bug in strict_rfc3339",
  157. subject="format",
  158. description="case-insensitive T and Z",
  159. )(test)
  160. ),
  161. )
  162. TestDraft7 = DRAFT7.to_unittest_testcase(
  163. DRAFT7.tests(),
  164. DRAFT7.format_tests(),
  165. DRAFT7.optional_tests_of(name="bignum"),
  166. DRAFT7.optional_tests_of(name="content"),
  167. DRAFT7.optional_tests_of(name="zeroTerminatedFloats"),
  168. Validator=Draft7Validator,
  169. format_checker=draft7_format_checker,
  170. skip=lambda test: (
  171. narrow_unicode_build(test)
  172. or missing_format(draft7_format_checker)(test)
  173. or skip(
  174. message=bug(),
  175. subject="ref",
  176. case_description="Recursive references between schemas",
  177. )(test)
  178. or skip(
  179. message=bug(371),
  180. subject="ref",
  181. case_description="Location-independent identifier",
  182. )(test)
  183. or skip(
  184. message=bug(371),
  185. subject="ref",
  186. case_description=(
  187. "Location-independent identifier with absolute URI"
  188. ),
  189. )(test)
  190. or skip(
  191. message=bug(371),
  192. subject="ref",
  193. case_description=(
  194. "Location-independent identifier with base URI change in subschema"
  195. ),
  196. )(test)
  197. or skip(
  198. message=bug(),
  199. subject="refRemote",
  200. case_description="base URI change - change folder in subschema",
  201. )(test)
  202. or skip(
  203. message="Upstream bug in strict_rfc3339",
  204. subject="date-time",
  205. description="case-insensitive T and Z",
  206. )(test)
  207. or skip(
  208. message=bug(593),
  209. subject="content",
  210. case_description=(
  211. "validation of string-encoded content based on media type"
  212. ),
  213. )(test)
  214. or skip(
  215. message=bug(593),
  216. subject="content",
  217. case_description="validation of binary string-encoding",
  218. )(test)
  219. or skip(
  220. message=bug(593),
  221. subject="content",
  222. case_description=(
  223. "validation of binary-encoded media type documents"
  224. ),
  225. )(test)
  226. ),
  227. )
  228. with warnings.catch_warnings():
  229. warnings.simplefilter("ignore", DeprecationWarning)
  230. TestDraft3LegacyTypeCheck = DRAFT3.to_unittest_testcase(
  231. # Interestingly the any part couldn't really be done w/the old API.
  232. (
  233. (test for test in each if test.schema != {"type": "any"})
  234. for each in DRAFT3.tests_of(name="type")
  235. ),
  236. name="TestDraft3LegacyTypeCheck",
  237. Validator=create(
  238. meta_schema=Draft3Validator.META_SCHEMA,
  239. validators=Draft3Validator.VALIDATORS,
  240. default_types=_DEPRECATED_DEFAULT_TYPES,
  241. ),
  242. )
  243. TestDraft4LegacyTypeCheck = DRAFT4.to_unittest_testcase(
  244. DRAFT4.tests_of(name="type"),
  245. name="TestDraft4LegacyTypeCheck",
  246. Validator=create(
  247. meta_schema=Draft4Validator.META_SCHEMA,
  248. validators=Draft4Validator.VALIDATORS,
  249. default_types=_DEPRECATED_DEFAULT_TYPES,
  250. ),
  251. )