default.py 1021 B

1234567891011121314151617181920212223242526272829303132
  1. from __future__ import absolute_import
  2. import random
  3. from .hashed import murmur2
  4. class DefaultPartitioner(object):
  5. """Default partitioner.
  6. Hashes key to partition using murmur2 hashing (from java client)
  7. If key is None, selects partition randomly from available,
  8. or from all partitions if none are currently available
  9. """
  10. @classmethod
  11. def __call__(cls, key, all_partitions, available):
  12. """
  13. Get the partition corresponding to key
  14. :param key: partitioning key
  15. :param all_partitions: list of all partitions sorted by partition ID
  16. :param available: list of available partitions in no particular order
  17. :return: one of the values from all_partitions or available
  18. """
  19. if key is None:
  20. if available:
  21. return random.choice(available)
  22. return random.choice(all_partitions)
  23. idx = murmur2(key)
  24. idx &= 0x7fffffff
  25. idx %= len(all_partitions)
  26. return all_partitions[idx]