lazy.py 5.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168
  1. from threading import RLock
  2. try:
  3. from UserDict import DictMixin
  4. except ImportError:
  5. from collections import Mapping as DictMixin
  6. # With lazy loading, we might end up with multiple threads triggering
  7. # it at the same time. We need a lock.
  8. _fill_lock = RLock()
  9. class LazyDict(DictMixin):
  10. """Dictionary populated on first use."""
  11. data = None
  12. def __getitem__(self, key):
  13. if self.data is None:
  14. _fill_lock.acquire()
  15. try:
  16. if self.data is None:
  17. self._fill()
  18. finally:
  19. _fill_lock.release()
  20. return self.data[key.upper()]
  21. def __contains__(self, key):
  22. if self.data is None:
  23. _fill_lock.acquire()
  24. try:
  25. if self.data is None:
  26. self._fill()
  27. finally:
  28. _fill_lock.release()
  29. return key in self.data
  30. def __iter__(self):
  31. if self.data is None:
  32. _fill_lock.acquire()
  33. try:
  34. if self.data is None:
  35. self._fill()
  36. finally:
  37. _fill_lock.release()
  38. return iter(self.data)
  39. def __len__(self):
  40. if self.data is None:
  41. _fill_lock.acquire()
  42. try:
  43. if self.data is None:
  44. self._fill()
  45. finally:
  46. _fill_lock.release()
  47. return len(self.data)
  48. def keys(self):
  49. if self.data is None:
  50. _fill_lock.acquire()
  51. try:
  52. if self.data is None:
  53. self._fill()
  54. finally:
  55. _fill_lock.release()
  56. return self.data.keys()
  57. class LazyList(list):
  58. """List populated on first use."""
  59. _props = [
  60. '__str__', '__repr__', '__unicode__',
  61. '__hash__', '__sizeof__', '__cmp__',
  62. '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__',
  63. 'append', 'count', 'index', 'extend', 'insert', 'pop', 'remove',
  64. 'reverse', 'sort', '__add__', '__radd__', '__iadd__', '__mul__',
  65. '__rmul__', '__imul__', '__contains__', '__len__', '__nonzero__',
  66. '__getitem__', '__setitem__', '__delitem__', '__iter__',
  67. '__reversed__', '__getslice__', '__setslice__', '__delslice__']
  68. def __new__(cls, fill_iter=None):
  69. if fill_iter is None:
  70. return list()
  71. # We need a new class as we will be dynamically messing with its
  72. # methods.
  73. class LazyList(list):
  74. pass
  75. fill_iter = [fill_iter]
  76. def lazy(name):
  77. def _lazy(self, *args, **kw):
  78. _fill_lock.acquire()
  79. try:
  80. if len(fill_iter) > 0:
  81. list.extend(self, fill_iter.pop())
  82. for method_name in cls._props:
  83. delattr(LazyList, method_name)
  84. finally:
  85. _fill_lock.release()
  86. return getattr(list, name)(self, *args, **kw)
  87. return _lazy
  88. for name in cls._props:
  89. setattr(LazyList, name, lazy(name))
  90. new_list = LazyList()
  91. return new_list
  92. # Not all versions of Python declare the same magic methods.
  93. # Filter out properties that don't exist in this version of Python
  94. # from the list.
  95. LazyList._props = [prop for prop in LazyList._props if hasattr(list, prop)]
  96. class LazySet(set):
  97. """Set populated on first use."""
  98. _props = (
  99. '__str__', '__repr__', '__unicode__',
  100. '__hash__', '__sizeof__', '__cmp__',
  101. '__lt__', '__le__', '__eq__', '__ne__', '__gt__', '__ge__',
  102. '__contains__', '__len__', '__nonzero__',
  103. '__getitem__', '__setitem__', '__delitem__', '__iter__',
  104. '__sub__', '__and__', '__xor__', '__or__',
  105. '__rsub__', '__rand__', '__rxor__', '__ror__',
  106. '__isub__', '__iand__', '__ixor__', '__ior__',
  107. 'add', 'clear', 'copy', 'difference', 'difference_update',
  108. 'discard', 'intersection', 'intersection_update', 'isdisjoint',
  109. 'issubset', 'issuperset', 'pop', 'remove',
  110. 'symmetric_difference', 'symmetric_difference_update',
  111. 'union', 'update')
  112. def __new__(cls, fill_iter=None):
  113. if fill_iter is None:
  114. return set()
  115. class LazySet(set):
  116. pass
  117. fill_iter = [fill_iter]
  118. def lazy(name):
  119. def _lazy(self, *args, **kw):
  120. _fill_lock.acquire()
  121. try:
  122. if len(fill_iter) > 0:
  123. for i in fill_iter.pop():
  124. set.add(self, i)
  125. for method_name in cls._props:
  126. delattr(LazySet, method_name)
  127. finally:
  128. _fill_lock.release()
  129. return getattr(set, name)(self, *args, **kw)
  130. return _lazy
  131. for name in cls._props:
  132. setattr(LazySet, name, lazy(name))
  133. new_set = LazySet()
  134. return new_set
  135. # Not all versions of Python declare the same magic methods.
  136. # Filter out properties that don't exist in this version of Python
  137. # from the list.
  138. LazySet._props = [prop for prop in LazySet._props if hasattr(set, prop)]