__init__.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. from __future__ import unicode_literals, absolute_import
  2. import io
  3. import os
  4. import re
  5. import abc
  6. import csv
  7. import sys
  8. import zipp
  9. import operator
  10. import functools
  11. import itertools
  12. import collections
  13. from ._compat import (
  14. install,
  15. NullFinder,
  16. ConfigParser,
  17. suppress,
  18. map,
  19. FileNotFoundError,
  20. IsADirectoryError,
  21. NotADirectoryError,
  22. PermissionError,
  23. pathlib,
  24. PYPY_OPEN_BUG,
  25. ModuleNotFoundError,
  26. MetaPathFinder,
  27. email_message_from_string,
  28. )
  29. from importlib import import_module
  30. from itertools import starmap
  31. __metaclass__ = type
  32. __all__ = [
  33. 'Distribution',
  34. 'PackageNotFoundError',
  35. 'distribution',
  36. 'distributions',
  37. 'entry_points',
  38. 'files',
  39. 'metadata',
  40. 'requires',
  41. 'version',
  42. ]
  43. class PackageNotFoundError(ModuleNotFoundError):
  44. """The package was not found."""
  45. class EntryPoint(collections.namedtuple('EntryPointBase', 'name value group')):
  46. """An entry point as defined by Python packaging conventions.
  47. See `the packaging docs on entry points
  48. <https://packaging.python.org/specifications/entry-points/>`_
  49. for more information.
  50. """
  51. pattern = re.compile(
  52. r'(?P<module>[\w.]+)\s*'
  53. r'(:\s*(?P<attr>[\w.]+))?\s*'
  54. r'(?P<extras>\[.*\])?\s*$'
  55. )
  56. """
  57. A regular expression describing the syntax for an entry point,
  58. which might look like:
  59. - module
  60. - package.module
  61. - package.module:attribute
  62. - package.module:object.attribute
  63. - package.module:attr [extra1, extra2]
  64. Other combinations are possible as well.
  65. The expression is lenient about whitespace around the ':',
  66. following the attr, and following any extras.
  67. """
  68. def load(self):
  69. """Load the entry point from its definition. If only a module
  70. is indicated by the value, return that module. Otherwise,
  71. return the named object.
  72. """
  73. match = self.pattern.match(self.value)
  74. module = import_module(match.group('module'))
  75. attrs = filter(None, (match.group('attr') or '').split('.'))
  76. return functools.reduce(getattr, attrs, module)
  77. @property
  78. def extras(self):
  79. match = self.pattern.match(self.value)
  80. return list(re.finditer(r'\w+', match.group('extras') or ''))
  81. @classmethod
  82. def _from_config(cls, config):
  83. return [
  84. cls(name, value, group)
  85. for group in config.sections()
  86. for name, value in config.items(group)
  87. ]
  88. @classmethod
  89. def _from_text(cls, text):
  90. config = ConfigParser()
  91. # case sensitive: https://stackoverflow.com/q/1611799/812183
  92. config.optionxform = str
  93. try:
  94. config.read_string(text)
  95. except AttributeError: # pragma: nocover
  96. # Python 2 has no read_string
  97. config.readfp(io.StringIO(text))
  98. return EntryPoint._from_config(config)
  99. def __iter__(self):
  100. """
  101. Supply iter so one may construct dicts of EntryPoints easily.
  102. """
  103. return iter((self.name, self))
  104. class PackagePath(pathlib.PurePosixPath):
  105. """A reference to a path in a package"""
  106. def read_text(self, encoding='utf-8'):
  107. with self.locate().open(encoding=encoding) as stream:
  108. return stream.read()
  109. def read_binary(self):
  110. with self.locate().open('rb') as stream:
  111. return stream.read()
  112. def locate(self):
  113. """Return a path-like object for this path"""
  114. return self.dist.locate_file(self)
  115. class FileHash:
  116. def __init__(self, spec):
  117. self.mode, _, self.value = spec.partition('=')
  118. def __repr__(self):
  119. return '<FileHash mode: {} value: {}>'.format(self.mode, self.value)
  120. class Distribution:
  121. """A Python distribution package."""
  122. @abc.abstractmethod
  123. def read_text(self, filename):
  124. """Attempt to load metadata file given by the name.
  125. :param filename: The name of the file in the distribution info.
  126. :return: The text if found, otherwise None.
  127. """
  128. @abc.abstractmethod
  129. def locate_file(self, path):
  130. """
  131. Given a path to a file in this distribution, return a path
  132. to it.
  133. """
  134. @classmethod
  135. def from_name(cls, name):
  136. """Return the Distribution for the given package name.
  137. :param name: The name of the distribution package to search for.
  138. :return: The Distribution instance (or subclass thereof) for the named
  139. package, if found.
  140. :raises PackageNotFoundError: When the named package's distribution
  141. metadata cannot be found.
  142. """
  143. for resolver in cls._discover_resolvers():
  144. dists = resolver(name)
  145. dist = next(dists, None)
  146. if dist is not None:
  147. return dist
  148. else:
  149. raise PackageNotFoundError(name)
  150. @classmethod
  151. def discover(cls):
  152. """Return an iterable of Distribution objects for all packages.
  153. :return: Iterable of Distribution objects for all packages.
  154. """
  155. return itertools.chain.from_iterable(
  156. resolver()
  157. for resolver in cls._discover_resolvers()
  158. )
  159. @staticmethod
  160. def _discover_resolvers():
  161. """Search the meta_path for resolvers."""
  162. declared = (
  163. getattr(finder, 'find_distributions', None)
  164. for finder in sys.meta_path
  165. )
  166. return filter(None, declared)
  167. @property
  168. def metadata(self):
  169. """Return the parsed metadata for this Distribution.
  170. The returned object will have keys that name the various bits of
  171. metadata. See PEP 566 for details.
  172. """
  173. text = (
  174. self.read_text('METADATA')
  175. or self.read_text('PKG-INFO')
  176. # This last clause is here to support old egg-info files. Its
  177. # effect is to just end up using the PathDistribution's self._path
  178. # (which points to the egg-info file) attribute unchanged.
  179. or self.read_text('')
  180. )
  181. return email_message_from_string(text)
  182. @property
  183. def version(self):
  184. """Return the 'Version' metadata for the distribution package."""
  185. return self.metadata['Version']
  186. @property
  187. def entry_points(self):
  188. return EntryPoint._from_text(self.read_text('entry_points.txt'))
  189. @property
  190. def files(self):
  191. file_lines = self._read_files_distinfo() or self._read_files_egginfo()
  192. def make_file(name, hash=None, size_str=None):
  193. result = PackagePath(name)
  194. result.hash = FileHash(hash) if hash else None
  195. result.size = int(size_str) if size_str else None
  196. result.dist = self
  197. return result
  198. return file_lines and starmap(make_file, csv.reader(file_lines))
  199. def _read_files_distinfo(self):
  200. """
  201. Read the lines of RECORD
  202. """
  203. text = self.read_text('RECORD')
  204. return text and text.splitlines()
  205. def _read_files_egginfo(self):
  206. """
  207. SOURCES.txt might contain literal commas, so wrap each line
  208. in quotes.
  209. """
  210. text = self.read_text('SOURCES.txt')
  211. return text and map('"{}"'.format, text.splitlines())
  212. @property
  213. def requires(self):
  214. """Generated requirements specified for this Distribution"""
  215. return self._read_dist_info_reqs() or self._read_egg_info_reqs()
  216. def _read_dist_info_reqs(self):
  217. spec = self.metadata['Requires-Dist']
  218. return spec and filter(None, spec.splitlines())
  219. def _read_egg_info_reqs(self):
  220. source = self.read_text('requires.txt')
  221. return source and self._deps_from_requires_text(source)
  222. @classmethod
  223. def _deps_from_requires_text(cls, source):
  224. section_pairs = cls._read_sections(source.splitlines())
  225. sections = {
  226. section: list(map(operator.itemgetter('line'), results))
  227. for section, results in
  228. itertools.groupby(section_pairs, operator.itemgetter('section'))
  229. }
  230. return cls._convert_egg_info_reqs_to_simple_reqs(sections)
  231. @staticmethod
  232. def _read_sections(lines):
  233. section = None
  234. for line in filter(None, lines):
  235. section_match = re.match(r'\[(.*)\]$', line)
  236. if section_match:
  237. section = section_match.group(1)
  238. continue
  239. yield locals()
  240. @staticmethod
  241. def _convert_egg_info_reqs_to_simple_reqs(sections):
  242. """
  243. Historically, setuptools would solicit and store 'extra'
  244. requirements, including those with environment markers,
  245. in separate sections. More modern tools expect each
  246. dependency to be defined separately, with any relevant
  247. extras and environment markers attached directly to that
  248. requirement. This method converts the former to the
  249. latter. See _test_deps_from_requires_text for an example.
  250. """
  251. def make_condition(name):
  252. return name and 'extra == "{name}"'.format(name=name)
  253. def parse_condition(section):
  254. section = section or ''
  255. extra, sep, markers = section.partition(':')
  256. if extra and markers:
  257. markers = '({markers})'.format(markers=markers)
  258. conditions = list(filter(None, [markers, make_condition(extra)]))
  259. return '; ' + ' and '.join(conditions) if conditions else ''
  260. for section, deps in sections.items():
  261. for dep in deps:
  262. yield dep + parse_condition(section)
  263. class DistributionFinder(MetaPathFinder):
  264. """
  265. A MetaPathFinder capable of discovering installed distributions.
  266. """
  267. @abc.abstractmethod
  268. def find_distributions(self, name=None, path=None):
  269. """
  270. Find distributions.
  271. Return an iterable of all Distribution instances capable of
  272. loading the metadata for packages matching the ``name``
  273. (or all names if not supplied) along the paths in the list
  274. of directories ``path`` (defaults to sys.path).
  275. """
  276. @install
  277. class MetadataPathFinder(NullFinder, DistributionFinder):
  278. """A degenerate finder for distribution packages on the file system.
  279. This finder supplies only a find_distributions() method for versions
  280. of Python that do not have a PathFinder find_distributions().
  281. """
  282. search_template = r'(?:{pattern}(-.*)?\.(dist|egg)-info|EGG-INFO)'
  283. def find_distributions(self, name=None, path=None):
  284. """
  285. Find distributions.
  286. Return an iterable of all Distribution instances capable of
  287. loading the metadata for packages matching the ``name``
  288. (or all names if not supplied) along the paths in the list
  289. of directories ``path`` (defaults to sys.path).
  290. """
  291. if path is None:
  292. path = sys.path
  293. pattern = '.*' if name is None else re.escape(name)
  294. found = self._search_paths(pattern, path)
  295. return map(PathDistribution, found)
  296. @classmethod
  297. def _search_paths(cls, pattern, paths):
  298. """Find metadata directories in paths heuristically."""
  299. return itertools.chain.from_iterable(
  300. cls._search_path(path, pattern)
  301. for path in map(cls._switch_path, paths)
  302. )
  303. @staticmethod
  304. def _switch_path(path):
  305. if not PYPY_OPEN_BUG or os.path.isfile(path): # pragma: no branch
  306. with suppress(Exception):
  307. return zipp.Path(path)
  308. return pathlib.Path(path)
  309. @classmethod
  310. def _predicate(cls, pattern, root, item):
  311. return re.match(pattern, str(item.name), flags=re.IGNORECASE)
  312. @classmethod
  313. def _search_path(cls, root, pattern):
  314. if not root.is_dir():
  315. return ()
  316. normalized = pattern.replace('-', '_')
  317. matcher = cls.search_template.format(pattern=normalized)
  318. return (item for item in root.iterdir()
  319. if cls._predicate(matcher, root, item))
  320. class PathDistribution(Distribution):
  321. def __init__(self, path):
  322. """Construct a distribution from a path to the metadata directory."""
  323. self._path = path
  324. def read_text(self, filename):
  325. with suppress(FileNotFoundError, IsADirectoryError, KeyError,
  326. NotADirectoryError, PermissionError):
  327. return self._path.joinpath(filename).read_text(encoding='utf-8')
  328. read_text.__doc__ = Distribution.read_text.__doc__
  329. def locate_file(self, path):
  330. return self._path.parent / path
  331. def distribution(package):
  332. """Get the ``Distribution`` instance for the given package.
  333. :param package: The name of the package as a string.
  334. :return: A ``Distribution`` instance (or subclass thereof).
  335. """
  336. return Distribution.from_name(package)
  337. def distributions():
  338. """Get all ``Distribution`` instances in the current environment.
  339. :return: An iterable of ``Distribution`` instances.
  340. """
  341. return Distribution.discover()
  342. def metadata(package):
  343. """Get the metadata for the package.
  344. :param package: The name of the distribution package to query.
  345. :return: An email.Message containing the parsed metadata.
  346. """
  347. return Distribution.from_name(package).metadata
  348. def version(package):
  349. """Get the version string for the named package.
  350. :param package: The name of the distribution package to query.
  351. :return: The version string for the package as defined in the package's
  352. "Version" metadata key.
  353. """
  354. return distribution(package).version
  355. def entry_points():
  356. """Return EntryPoint objects for all installed packages.
  357. :return: EntryPoint objects for all installed packages.
  358. """
  359. eps = itertools.chain.from_iterable(
  360. dist.entry_points for dist in distributions())
  361. by_group = operator.attrgetter('group')
  362. ordered = sorted(eps, key=by_group)
  363. grouped = itertools.groupby(ordered, by_group)
  364. return {
  365. group: tuple(eps)
  366. for group, eps in grouped
  367. }
  368. def files(package):
  369. return distribution(package).files
  370. def requires(package):
  371. """
  372. Return a list of requirements for the indicated distribution.
  373. :return: An iterator of requirements, suitable for
  374. packaging.requirement.Requirement.
  375. """
  376. return distribution(package).requires
  377. __version__ = version(__name__)