manyi/library/pymemcache/serde.py

# Copyright 2012 Pinterest.com
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from functools import partial
import logging
from io import BytesIO
import six
from six.moves import cPickle as pickle

try:
    long_type = long  # noqa
except NameError:
    long_type = None

FLAG_COMPRESSED = 1 << 3  # unused, to main compatibility with python-memcached
FLAG_PICKLE = 1 << 0
FLAG_UNICODE = 1 << 6

# to main compatibility with java memcache
FLAG_INTEGER = 1 << 2
FLAG_INTEGER_OLD = 1 << 1
FLAG_STRING = 1 << 5
FLAG_LONG = 1 << 14

# Pickle protocol version (highest available to runtime)
# Warning with `0`: If somewhere in your value lies a slotted object,
# ie defines `__slots__`, even if you do not include it in your pickleable
# state via `__getstate__`, python will complain with something like:
#   TypeError: a class that defines __slots__ without defining __getstate__
#   cannot be pickled
DEFAULT_PICKLE_VERSION = pickle.HIGHEST_PROTOCOL


def _python_memcache_serializer(key, value, pickle_version=None):
    flags = 0

    value_type = type(value)

    # Check against exact types so that subclasses of native types will be
    # restored as their native type

    if value_type is bytes:
        flags |= FLAG_STRING

    elif value_type is six.text_type:
        flags |= FLAG_UNICODE
        value = value.encode('utf8')

    elif value_type is int:
        flags |= FLAG_INTEGER
        value = "%d" % value

    elif six.PY2 and value_type is long_type:
        flags |= FLAG_LONG
        value = "%d" % value

    else:
        flags |= FLAG_PICKLE
        output = BytesIO()
        pickler = pickle.Pickler(output, pickle_version)
        pickler.dump(value)
        value = output.getvalue()

    return value, flags


def get_python_memcache_serializer(pickle_version=DEFAULT_PICKLE_VERSION):
    """Return a serializer using a specific pickle version"""
    return partial(_python_memcache_serializer, pickle_version=pickle_version)


python_memcache_serializer = get_python_memcache_serializer()


def python_memcache_deserializer(key, value, flags):
    if flags == 0 or flags == FLAG_STRING:
        return value

    elif flags & FLAG_UNICODE:
        return value.decode('utf8')

    elif flags & FLAG_INTEGER:
        return int(value)

    elif flags & FLAG_INTEGER_OLD:
        return int(value)

    elif flags & FLAG_LONG:
        if six.PY3:
            return int(value)
        else:
            return long_type(value)

    elif flags & FLAG_PICKLE:
        try:
            buf = BytesIO(value)
            unpickler = pickle.Unpickler(buf)
            return unpickler.load()
        except Exception:
            logging.info('Pickle error', exc_info=True)
            return None

    return value


class PickleSerde(object):
    """
    An object which implements the serialization/deserialization protocol for
    :py:class:`pymemcache.client.base.Client` and its descendants using the
    :mod:`pickle` module.

    Serialization and deserialization are implemented as methods of this class.
    To implement a custom serialization/deserialization method for pymemcache,
    you should implement the same interface as the one provided by this object
    -- :py:meth:`pymemcache.serde.PickleSerde.serialize` and
    :py:meth:`pymemcache.serde.PickleSerde.deserialize`. Then,
    pass your custom object to the pymemcache client object in place of
    `PickleSerde`.

    For more details on the serialization protocol, see the class documentation
    for :py:class:`pymemcache.client.base.Client`
    """
    def __init__(self, pickle_version=DEFAULT_PICKLE_VERSION):
        self._serialize_func = get_python_memcache_serializer(pickle_version)

    def serialize(self, key, value):
        return self._serialize_func(key, value)

    def deserialize(self, key, value, flags):
        return python_memcache_deserializer(key, value, flags)


class LegacyWrappingSerde(object):
    """
    This class defines how to wrap legacy de/serialization functions into a
    'serde' object which implements '.serialize' and '.deserialize' methods.
    It is used automatically by pymemcache.client.base.Client when the
    'serializer' or 'deserializer' arguments are given.

    The serializer_func and deserializer_func are expected to be None in the
    case that they are missing.
    """
    def __init__(self, serializer_func, deserializer_func):
        self.serialize = serializer_func or self._default_serialize
        self.deserialize = deserializer_func or self._default_deserialize

    def _default_serialize(self, key, value):
        return value, 0

    def _default_deserialize(self, key, value, flags):
        return value


pickle_serde = PickleSerde()