localedata.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224
  1. # -*- coding: utf-8 -*-
  2. """
  3. babel.localedata
  4. ~~~~~~~~~~~~~~~~
  5. Low-level locale data access.
  6. :note: The `Locale` class, which uses this module under the hood, provides a
  7. more convenient interface for accessing the locale data.
  8. :copyright: (c) 2013 by the Babel Team.
  9. :license: BSD, see LICENSE for more details.
  10. """
  11. import os
  12. import threading
  13. from collections import MutableMapping
  14. from itertools import chain
  15. from babel._compat import pickle
  16. _cache = {}
  17. _cache_lock = threading.RLock()
  18. _dirname = os.path.join(os.path.dirname(__file__), 'locale-data')
  19. def normalize_locale(name):
  20. """Normalize a locale ID by stripping spaces and apply proper casing.
  21. Returns the normalized locale ID string or `None` if the ID is not
  22. recognized.
  23. """
  24. name = name.strip().lower()
  25. for locale_id in chain.from_iterable([_cache, locale_identifiers()]):
  26. if name == locale_id.lower():
  27. return locale_id
  28. def exists(name):
  29. """Check whether locale data is available for the given locale.
  30. Returns `True` if it exists, `False` otherwise.
  31. :param name: the locale identifier string
  32. """
  33. if name in _cache:
  34. return True
  35. file_found = os.path.exists(os.path.join(_dirname, '%s.dat' % name))
  36. return True if file_found else bool(normalize_locale(name))
  37. def locale_identifiers():
  38. """Return a list of all locale identifiers for which locale data is
  39. available.
  40. .. versionadded:: 0.8.1
  41. :return: a list of locale identifiers (strings)
  42. """
  43. return [stem for stem, extension in [
  44. os.path.splitext(filename) for filename in os.listdir(_dirname)
  45. ] if extension == '.dat' and stem != 'root']
  46. def load(name, merge_inherited=True):
  47. """Load the locale data for the given locale.
  48. The locale data is a dictionary that contains much of the data defined by
  49. the Common Locale Data Repository (CLDR). This data is stored as a
  50. collection of pickle files inside the ``babel`` package.
  51. >>> d = load('en_US')
  52. >>> d['languages']['sv']
  53. u'Swedish'
  54. Note that the results are cached, and subsequent requests for the same
  55. locale return the same dictionary:
  56. >>> d1 = load('en_US')
  57. >>> d2 = load('en_US')
  58. >>> d1 is d2
  59. True
  60. :param name: the locale identifier string (or "root")
  61. :param merge_inherited: whether the inherited data should be merged into
  62. the data of the requested locale
  63. :raise `IOError`: if no locale data file is found for the given locale
  64. identifer, or one of the locales it inherits from
  65. """
  66. _cache_lock.acquire()
  67. try:
  68. data = _cache.get(name)
  69. if not data:
  70. # Load inherited data
  71. if name == 'root' or not merge_inherited:
  72. data = {}
  73. else:
  74. from babel.core import get_global
  75. parent = get_global('parent_exceptions').get(name)
  76. if not parent:
  77. parts = name.split('_')
  78. if len(parts) == 1:
  79. parent = 'root'
  80. else:
  81. parent = '_'.join(parts[:-1])
  82. data = load(parent).copy()
  83. filename = os.path.join(_dirname, '%s.dat' % name)
  84. with open(filename, 'rb') as fileobj:
  85. if name != 'root' and merge_inherited:
  86. merge(data, pickle.load(fileobj))
  87. else:
  88. data = pickle.load(fileobj)
  89. _cache[name] = data
  90. return data
  91. finally:
  92. _cache_lock.release()
  93. def merge(dict1, dict2):
  94. """Merge the data from `dict2` into the `dict1` dictionary, making copies
  95. of nested dictionaries.
  96. >>> d = {1: 'foo', 3: 'baz'}
  97. >>> merge(d, {1: 'Foo', 2: 'Bar'})
  98. >>> sorted(d.items())
  99. [(1, 'Foo'), (2, 'Bar'), (3, 'baz')]
  100. :param dict1: the dictionary to merge into
  101. :param dict2: the dictionary containing the data that should be merged
  102. """
  103. for key, val2 in dict2.items():
  104. if val2 is not None:
  105. val1 = dict1.get(key)
  106. if isinstance(val2, dict):
  107. if val1 is None:
  108. val1 = {}
  109. if isinstance(val1, Alias):
  110. val1 = (val1, val2)
  111. elif isinstance(val1, tuple):
  112. alias, others = val1
  113. others = others.copy()
  114. merge(others, val2)
  115. val1 = (alias, others)
  116. else:
  117. val1 = val1.copy()
  118. merge(val1, val2)
  119. else:
  120. val1 = val2
  121. dict1[key] = val1
  122. class Alias(object):
  123. """Representation of an alias in the locale data.
  124. An alias is a value that refers to some other part of the locale data,
  125. as specified by the `keys`.
  126. """
  127. def __init__(self, keys):
  128. self.keys = tuple(keys)
  129. def __repr__(self):
  130. return '<%s %r>' % (type(self).__name__, self.keys)
  131. def resolve(self, data):
  132. """Resolve the alias based on the given data.
  133. This is done recursively, so if one alias resolves to a second alias,
  134. that second alias will also be resolved.
  135. :param data: the locale data
  136. :type data: `dict`
  137. """
  138. base = data
  139. for key in self.keys:
  140. data = data[key]
  141. if isinstance(data, Alias):
  142. data = data.resolve(base)
  143. elif isinstance(data, tuple):
  144. alias, others = data
  145. data = alias.resolve(base)
  146. return data
  147. class LocaleDataDict(MutableMapping):
  148. """Dictionary wrapper that automatically resolves aliases to the actual
  149. values.
  150. """
  151. def __init__(self, data, base=None):
  152. self._data = data
  153. if base is None:
  154. base = data
  155. self.base = base
  156. def __len__(self):
  157. return len(self._data)
  158. def __iter__(self):
  159. return iter(self._data)
  160. def __getitem__(self, key):
  161. orig = val = self._data[key]
  162. if isinstance(val, Alias): # resolve an alias
  163. val = val.resolve(self.base)
  164. if isinstance(val, tuple): # Merge a partial dict with an alias
  165. alias, others = val
  166. val = alias.resolve(self.base).copy()
  167. merge(val, others)
  168. if type(val) is dict: # Return a nested alias-resolving dict
  169. val = LocaleDataDict(val, base=self.base)
  170. if val is not orig:
  171. self._data[key] = val
  172. return val
  173. def __setitem__(self, key, value):
  174. self._data[key] = value
  175. def __delitem__(self, key):
  176. del self._data[key]
  177. def copy(self):
  178. return LocaleDataDict(self._data.copy(), base=self.base)