resolvers.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454
  1. import collections
  2. from .providers import AbstractResolver
  3. from .structs import DirectedGraph, build_iter_view
  4. RequirementInformation = collections.namedtuple(
  5. "RequirementInformation", ["requirement", "parent"]
  6. )
  7. class ResolverException(Exception):
  8. """A base class for all exceptions raised by this module.
  9. Exceptions derived by this class should all be handled in this module. Any
  10. bubbling pass the resolver should be treated as a bug.
  11. """
  12. class RequirementsConflicted(ResolverException):
  13. def __init__(self, criterion):
  14. super(RequirementsConflicted, self).__init__(criterion)
  15. self.criterion = criterion
  16. def __str__(self):
  17. return "Requirements conflict: {}".format(
  18. ", ".join(repr(r) for r in self.criterion.iter_requirement()),
  19. )
  20. class InconsistentCandidate(ResolverException):
  21. def __init__(self, candidate, criterion):
  22. super(InconsistentCandidate, self).__init__(candidate, criterion)
  23. self.candidate = candidate
  24. self.criterion = criterion
  25. def __str__(self):
  26. return "Provided candidate {!r} does not satisfy {}".format(
  27. self.candidate,
  28. ", ".join(repr(r) for r in self.criterion.iter_requirement()),
  29. )
  30. class Criterion(object):
  31. """Representation of possible resolution results of a package.
  32. This holds three attributes:
  33. * `information` is a collection of `RequirementInformation` pairs.
  34. Each pair is a requirement contributing to this criterion, and the
  35. candidate that provides the requirement.
  36. * `incompatibilities` is a collection of all known not-to-work candidates
  37. to exclude from consideration.
  38. * `candidates` is a collection containing all possible candidates deducted
  39. from the union of contributing requirements and known incompatibilities.
  40. It should never be empty, except when the criterion is an attribute of a
  41. raised `RequirementsConflicted` (in which case it is always empty).
  42. .. note::
  43. This class is intended to be externally immutable. **Do not** mutate
  44. any of its attribute containers.
  45. """
  46. def __init__(self, candidates, information, incompatibilities):
  47. self.candidates = candidates
  48. self.information = information
  49. self.incompatibilities = incompatibilities
  50. def __repr__(self):
  51. requirements = ", ".join(
  52. "({!r}, via={!r})".format(req, parent)
  53. for req, parent in self.information
  54. )
  55. return "Criterion({})".format(requirements)
  56. @classmethod
  57. def from_requirement(cls, provider, requirement, parent):
  58. """Build an instance from a requirement."""
  59. cands = build_iter_view(provider.find_matches([requirement]))
  60. infos = [RequirementInformation(requirement, parent)]
  61. criterion = cls(cands, infos, incompatibilities=[])
  62. if not cands:
  63. raise RequirementsConflicted(criterion)
  64. return criterion
  65. def iter_requirement(self):
  66. return (i.requirement for i in self.information)
  67. def iter_parent(self):
  68. return (i.parent for i in self.information)
  69. def merged_with(self, provider, requirement, parent):
  70. """Build a new instance from this and a new requirement."""
  71. infos = list(self.information)
  72. infos.append(RequirementInformation(requirement, parent))
  73. cands = build_iter_view(provider.find_matches([r for r, _ in infos]))
  74. criterion = type(self)(cands, infos, list(self.incompatibilities))
  75. if not cands:
  76. raise RequirementsConflicted(criterion)
  77. return criterion
  78. def excluded_of(self, candidates):
  79. """Build a new instance from this, but excluding specified candidates.
  80. Returns the new instance, or None if we still have no valid candidates.
  81. """
  82. cands = self.candidates.excluding(candidates)
  83. if not cands:
  84. return None
  85. incompats = self.incompatibilities + candidates
  86. return type(self)(cands, list(self.information), incompats)
  87. class ResolutionError(ResolverException):
  88. pass
  89. class ResolutionImpossible(ResolutionError):
  90. def __init__(self, causes):
  91. super(ResolutionImpossible, self).__init__(causes)
  92. # causes is a list of RequirementInformation objects
  93. self.causes = causes
  94. class ResolutionTooDeep(ResolutionError):
  95. def __init__(self, round_count):
  96. super(ResolutionTooDeep, self).__init__(round_count)
  97. self.round_count = round_count
  98. # Resolution state in a round.
  99. State = collections.namedtuple("State", "mapping criteria")
  100. class Resolution(object):
  101. """Stateful resolution object.
  102. This is designed as a one-off object that holds information to kick start
  103. the resolution process, and holds the results afterwards.
  104. """
  105. def __init__(self, provider, reporter):
  106. self._p = provider
  107. self._r = reporter
  108. self._states = []
  109. @property
  110. def state(self):
  111. try:
  112. return self._states[-1]
  113. except IndexError:
  114. raise AttributeError("state")
  115. def _push_new_state(self):
  116. """Push a new state into history.
  117. This new state will be used to hold resolution results of the next
  118. coming round.
  119. """
  120. base = self._states[-1]
  121. state = State(
  122. mapping=base.mapping.copy(),
  123. criteria=base.criteria.copy(),
  124. )
  125. self._states.append(state)
  126. def _merge_into_criterion(self, requirement, parent):
  127. self._r.adding_requirement(requirement, parent)
  128. name = self._p.identify(requirement)
  129. try:
  130. crit = self.state.criteria[name]
  131. except KeyError:
  132. crit = Criterion.from_requirement(self._p, requirement, parent)
  133. else:
  134. crit = crit.merged_with(self._p, requirement, parent)
  135. return name, crit
  136. def _get_criterion_item_preference(self, item):
  137. name, criterion = item
  138. return self._p.get_preference(
  139. self.state.mapping.get(name),
  140. criterion.candidates.for_preference(),
  141. criterion.information,
  142. )
  143. def _is_current_pin_satisfying(self, name, criterion):
  144. try:
  145. current_pin = self.state.mapping[name]
  146. except KeyError:
  147. return False
  148. return all(
  149. self._p.is_satisfied_by(r, current_pin)
  150. for r in criterion.iter_requirement()
  151. )
  152. def _get_criteria_to_update(self, candidate):
  153. criteria = {}
  154. for r in self._p.get_dependencies(candidate):
  155. name, crit = self._merge_into_criterion(r, parent=candidate)
  156. criteria[name] = crit
  157. return criteria
  158. def _attempt_to_pin_criterion(self, name, criterion):
  159. causes = []
  160. for candidate in criterion.candidates:
  161. try:
  162. criteria = self._get_criteria_to_update(candidate)
  163. except RequirementsConflicted as e:
  164. causes.append(e.criterion)
  165. continue
  166. # Check the newly-pinned candidate actually works. This should
  167. # always pass under normal circumstances, but in the case of a
  168. # faulty provider, we will raise an error to notify the implementer
  169. # to fix find_matches() and/or is_satisfied_by().
  170. satisfied = all(
  171. self._p.is_satisfied_by(r, candidate)
  172. for r in criterion.iter_requirement()
  173. )
  174. if not satisfied:
  175. raise InconsistentCandidate(candidate, criterion)
  176. # Put newly-pinned candidate at the end. This is essential because
  177. # backtracking looks at this mapping to get the last pin.
  178. self._r.pinning(candidate)
  179. self.state.mapping.pop(name, None)
  180. self.state.mapping[name] = candidate
  181. self.state.criteria.update(criteria)
  182. return []
  183. # All candidates tried, nothing works. This criterion is a dead
  184. # end, signal for backtracking.
  185. return causes
  186. def _backtrack(self):
  187. """Perform backtracking.
  188. When we enter here, the stack is like this::
  189. [ state Z ]
  190. [ state Y ]
  191. [ state X ]
  192. .... earlier states are irrelevant.
  193. 1. No pins worked for Z, so it does not have a pin.
  194. 2. We want to reset state Y to unpinned, and pin another candidate.
  195. 3. State X holds what state Y was before the pin, but does not
  196. have the incompatibility information gathered in state Y.
  197. Each iteration of the loop will:
  198. 1. Discard Z.
  199. 2. Discard Y but remember its incompatibility information gathered
  200. previously, and the failure we're dealing with right now.
  201. 3. Push a new state Y' based on X, and apply the incompatibility
  202. information from Y to Y'.
  203. 4a. If this causes Y' to conflict, we need to backtrack again. Make Y'
  204. the new Z and go back to step 2.
  205. 4b. If the incompatibilities apply cleanly, end backtracking.
  206. """
  207. while len(self._states) >= 3:
  208. # Remove the state that triggered backtracking.
  209. del self._states[-1]
  210. # Retrieve the last candidate pin and known incompatibilities.
  211. broken_state = self._states.pop()
  212. name, candidate = broken_state.mapping.popitem()
  213. incompatibilities_from_broken = [
  214. (k, v.incompatibilities)
  215. for k, v in broken_state.criteria.items()
  216. ]
  217. # Also mark the newly known incompatibility.
  218. incompatibilities_from_broken.append((name, [candidate]))
  219. self._r.backtracking(candidate)
  220. # Create a new state from the last known-to-work one, and apply
  221. # the previously gathered incompatibility information.
  222. def _patch_criteria():
  223. for k, incompatibilities in incompatibilities_from_broken:
  224. if not incompatibilities:
  225. continue
  226. try:
  227. criterion = self.state.criteria[k]
  228. except KeyError:
  229. continue
  230. criterion = criterion.excluded_of(incompatibilities)
  231. if criterion is None:
  232. return False
  233. self.state.criteria[k] = criterion
  234. return True
  235. self._push_new_state()
  236. success = _patch_criteria()
  237. # It works! Let's work on this new state.
  238. if success:
  239. return True
  240. # State does not work after applying known incompatibilities.
  241. # Try the still previous state.
  242. # No way to backtrack anymore.
  243. return False
  244. def resolve(self, requirements, max_rounds):
  245. if self._states:
  246. raise RuntimeError("already resolved")
  247. self._r.starting()
  248. # Initialize the root state.
  249. self._states = [State(mapping=collections.OrderedDict(), criteria={})]
  250. for r in requirements:
  251. try:
  252. name, crit = self._merge_into_criterion(r, parent=None)
  253. except RequirementsConflicted as e:
  254. raise ResolutionImpossible(e.criterion.information)
  255. self.state.criteria[name] = crit
  256. # The root state is saved as a sentinel so the first ever pin can have
  257. # something to backtrack to if it fails. The root state is basically
  258. # pinning the virtual "root" package in the graph.
  259. self._push_new_state()
  260. for round_index in range(max_rounds):
  261. self._r.starting_round(round_index)
  262. unsatisfied_criterion_items = [
  263. item
  264. for item in self.state.criteria.items()
  265. if not self._is_current_pin_satisfying(*item)
  266. ]
  267. # All criteria are accounted for. Nothing more to pin, we are done!
  268. if not unsatisfied_criterion_items:
  269. self._r.ending(self.state)
  270. return self.state
  271. # Choose the most preferred unpinned criterion to try.
  272. name, criterion = min(
  273. unsatisfied_criterion_items,
  274. key=self._get_criterion_item_preference,
  275. )
  276. failure_causes = self._attempt_to_pin_criterion(name, criterion)
  277. if failure_causes:
  278. # Backtrack if pinning fails. The backtrack process puts us in
  279. # an unpinned state, so we can work on it in the next round.
  280. success = self._backtrack()
  281. # Dead ends everywhere. Give up.
  282. if not success:
  283. causes = [i for c in failure_causes for i in c.information]
  284. raise ResolutionImpossible(causes)
  285. else:
  286. # Pinning was successful. Push a new state to do another pin.
  287. self._push_new_state()
  288. self._r.ending_round(round_index, self.state)
  289. raise ResolutionTooDeep(max_rounds)
  290. def _has_route_to_root(criteria, key, all_keys, connected):
  291. if key in connected:
  292. return True
  293. if key not in criteria:
  294. return False
  295. for p in criteria[key].iter_parent():
  296. try:
  297. pkey = all_keys[id(p)]
  298. except KeyError:
  299. continue
  300. if pkey in connected:
  301. connected.add(key)
  302. return True
  303. if _has_route_to_root(criteria, pkey, all_keys, connected):
  304. connected.add(key)
  305. return True
  306. return False
  307. Result = collections.namedtuple("Result", "mapping graph criteria")
  308. def _build_result(state):
  309. mapping = state.mapping
  310. all_keys = {id(v): k for k, v in mapping.items()}
  311. all_keys[id(None)] = None
  312. graph = DirectedGraph()
  313. graph.add(None) # Sentinel as root dependencies' parent.
  314. connected = {None}
  315. for key, criterion in state.criteria.items():
  316. if not _has_route_to_root(state.criteria, key, all_keys, connected):
  317. continue
  318. if key not in graph:
  319. graph.add(key)
  320. for p in criterion.iter_parent():
  321. try:
  322. pkey = all_keys[id(p)]
  323. except KeyError:
  324. continue
  325. if pkey not in graph:
  326. graph.add(pkey)
  327. graph.connect(pkey, key)
  328. return Result(
  329. mapping={k: v for k, v in mapping.items() if k in connected},
  330. graph=graph,
  331. criteria=state.criteria,
  332. )
  333. class Resolver(AbstractResolver):
  334. """The thing that performs the actual resolution work."""
  335. base_exception = ResolverException
  336. def resolve(self, requirements, max_rounds=100):
  337. """Take a collection of constraints, spit out the resolution result.
  338. The return value is a representation to the final resolution result. It
  339. is a tuple subclass with three public members:
  340. * `mapping`: A dict of resolved candidates. Each key is an identifier
  341. of a requirement (as returned by the provider's `identify` method),
  342. and the value is the resolved candidate.
  343. * `graph`: A `DirectedGraph` instance representing the dependency tree.
  344. The vertices are keys of `mapping`, and each edge represents *why*
  345. a particular package is included. A special vertex `None` is
  346. included to represent parents of user-supplied requirements.
  347. * `criteria`: A dict of "criteria" that hold detailed information on
  348. how edges in the graph are derived. Each key is an identifier of a
  349. requirement, and the value is a `Criterion` instance.
  350. The following exceptions may be raised if a resolution cannot be found:
  351. * `ResolutionImpossible`: A resolution cannot be found for the given
  352. combination of requirements. The `causes` attribute of the
  353. exception is a list of (requirement, parent), giving the
  354. requirements that could not be satisfied.
  355. * `ResolutionTooDeep`: The dependency tree is too deeply nested and
  356. the resolver gave up. This is usually caused by a circular
  357. dependency, but you can try to resolve this by increasing the
  358. `max_rounds` argument.
  359. """
  360. resolution = Resolution(self.provider, self.reporter)
  361. state = resolution.resolve(requirements, max_rounds=max_rounds)
  362. return _build_result(state)