# -*- coding: utf-8 -*- from . import http from . import exceptions from . import Bucket from .api import _make_range_string from .models import * from .compat import to_string, urlsplit, parse_qs from .crypto import BaseCryptoProvider from .exceptions import ClientError import copy import threading logger = logging.getLogger(__name__) class CryptoBucket(Bucket): """用于加密Bucket和Object操作的类，诸如上传、下载Object等。创建、删除bucket的操作需使用Bucket类接口。 用法（假设Bucket属于杭州区域） :: >>> import oss2 >>> auth = oss2.Auth('your-access-key-id', 'your-access-key-secret') >>> bucket = oss2.CryptoBucket(auth, 'http://oss-cn-hangzhou.aliyuncs.com', 'your-bucket', oss2.LocalRsaProvider()) >>> bucket.put_object('readme.txt', 'content of the object') :param auth: 包含了用户认证信息的Auth对象 :type auth: oss2.Auth :param str endpoint: 访问域名或者CNAME :param str bucket_name: Bucket名 :param crypto_provider: 客户端加密类。该参数默认为空 :type crypto_provider: oss2.crypto.BaseCryptoProvider :param bool is_cname: 如果endpoint是CNAME则设为True；反之，则为False。 :param session: 会话。如果是None表示新开会话，非None则复用传入的会话 :type session: oss2.Session :param float connect_timeout: 连接超时时间，以秒为单位。 :param str app_name: 应用名。该参数不为空，则在User Agent中加入其值。 注意到，最终这个字符串是要作为HTTP Header的值传输的，所以必须要遵循HTTP标准。 :param bool enable_crc: 如果开启crc校验则设为True；反之，则为False """ def __init__(self, auth, endpoint, bucket_name, crypto_provider, is_cname=False, session=None, connect_timeout=None, app_name='', enable_crc=True, ): if not isinstance(crypto_provider, BaseCryptoProvider): raise ClientError('crypto_provider must be an instance of BaseCryptoProvider') logger.debug("Init CryptoBucket: {0}".format(bucket_name)) super(CryptoBucket, self).__init__(auth, endpoint, bucket_name, is_cname, session, connect_timeout, app_name, enable_crc) self.crypto_provider = crypto_provider self.upload_contexts = {} self.upload_contexts_lock = threading.Lock() if self.app_name: self.user_agent = http.USER_AGENT + '/' + self.app_name + '/' + OSS_ENCRYPTION_CLIENT else: self.user_agent = http.USER_AGENT + '/' + OSS_ENCRYPTION_CLIENT def _init_user_agent(self, headers): if 'User-Agent' not in headers: headers['User-Agent'] = self.user_agent else: headers['User-Agent'] += '/' + OSS_ENCRYPTION_CLIENT def put_object(self, key, data, headers=None, progress_callback=None): """上传一个普通文件。 用法 :: >>> bucket.put_object('readme.txt', 'content of readme.txt') >>> with open(u'local_file.txt', 'rb') as f: >>> bucket.put_object('remote_file.txt', f) :param mat_desc: map，对象文件的description :param key: 上传到OSS的文件名 :param data: 待上传的内容。 :type data: bytes，str或file-like object :param headers: 用户指定的HTTP头部。可以指定Content-Type、Content-MD5、x-oss-meta-开头的头部等 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :param progress_callback: 用户指定的进度回调函数。可以用来实现进度条等功能。参考 :ref:`progress_callback` 。 :return: :class:`PutObjectResult ` """ logger.debug("Start to put object to CryptoBucket") headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) content_crypto_material = self.crypto_provider.create_content_material() data = self.crypto_provider.make_encrypt_adapter(data, content_crypto_material.cipher) headers = content_crypto_material.to_object_meta(headers) return super(CryptoBucket, self).put_object(key, data, headers, progress_callback) def put_object_with_url(self, sign_url, data, headers=None, progress_callback=None): """ 使用加签的url上传对象 :param sign_url: 加签的url :param data: 待上传的数据 :param headers: 用户指定的HTTP头部。可以指定Content-Type、Content-MD5、x-oss-meta-开头的头部等，必须和签名时保持一致 :param progress_callback: 用户指定的进度回调函数。参考 :ref:`progress_callback` :return: """ raise ClientError("The operation is not support for CryptoBucket now") def append_object(self, key, position, data, headers=None, progress_callback=None, init_crc=None): raise ClientError("The operation is not support for CryptoBucket") def get_object(self, key, byte_range=None, headers=None, progress_callback=None, process=None, params=None): """下载一个文件。 用法 :: >>> result = bucket.get_object('readme.txt') >>> print(result.read()) 'hello world' :param key: 文件名 :param byte_range: 指定下载范围。参见 :ref:`byte_range` :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :param progress_callback: 用户指定的进度回调函数。参考 :ref:`progress_callback` :param process: oss文件处理，如图像服务等。指定后process，返回的内容为处理后的文件。 :param params: http 请求的查询字符串参数 :type params: dict :return: file-like object :raises: 如果文件不存在，则抛出 :class:`NoSuchKey ` ；还可能抛出其他异常 """ if process: raise ClientError("Process object operation is not support for Crypto Bucket") headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) discard = 0 range_string = '' if byte_range: if byte_range[0] is None and byte_range[1]: raise ClientError("Don't support range get while start is none and end is not") start, end = self.crypto_provider.adjust_range(byte_range[0], byte_range[1]) adjust_byte_range = (start, end) range_string = _make_range_string(adjust_byte_range) if range_string: headers['range'] = range_string if byte_range[0] and adjust_byte_range[0] < byte_range[0]: discard = byte_range[0] - adjust_byte_range[0] logger.debug("adjust range of get object, byte_range: {0}, adjust_byte_range: {1}, discard: {2}".format( byte_range, adjust_byte_range, discard)) logger.debug( "Start to get object from CryptoBucket: {0}, key: {1}, range: {2}, headers: {3}, params: {4}".format( self.bucket_name, to_string(key), range_string, headers, params)) resp = self._do('GET', self.bucket_name, key, headers=headers, params=params) logger.debug("Get object done, req_id: {0}, status_code: {1}".format(resp.request_id, resp.status)) return GetObjectResult(resp, progress_callback, self.enable_crc, crypto_provider=self.crypto_provider, discard=discard) def get_object_with_url(self, sign_url, byte_range=None, headers=None, progress_callback=None): """使用加签的url下载文件 :param sign_url: 加签的url :param byte_range: 指定下载范围。参见 :ref:`byte_range` :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict，必须和签名时保持一致 :param progress_callback: 用户指定的进度回调函数。参考 :ref:`progress_callback` :return: file-like object :raises: 如果文件不存在，则抛出 :class:`NoSuchKey ` ；还可能抛出其他异常 """ query = parse_qs(urlsplit(sign_url).query) if query and (Bucket.PROCESS in query): raise ClientError("Process object operation is not support for Crypto Bucket") headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) discard = 0 range_string = '' if byte_range: if not byte_range[0] and byte_range[1]: raise ClientError("Don't support range get while start is none and end is not") start, end = self.crypto_provider.adjust_range(byte_range[0], byte_range[1]) adjust_byte_range = (start, end) range_string = _make_range_string(adjust_byte_range) if range_string: headers['range'] = range_string if byte_range[0] and adjust_byte_range[0] < byte_range[0]: discard = byte_range[0] - adjust_byte_range[0] logger.debug("adjust range of get object, byte_range: {0}, adjust_byte_range: {1}, discard: {2}".format( byte_range, adjust_byte_range, discard)) logger.debug( "Start to get object with url from CryptoBucket: {0}, sign_url: {1}, range: {2}, headers: {3}".format( self.bucket_name, sign_url, range_string, headers)) resp = self._do_url('GET', sign_url, headers=headers) return GetObjectResult(resp, progress_callback, self.enable_crc, crypto_provider=self.crypto_provider, discard=discard) def create_select_object_meta(self, key, select_meta_params=None): raise ClientError("The operation is not support for Crypto Bucket") def select_object(self, key, sql, progress_callback=None, select_params=None ): raise ClientError("The operation is not support for CryptoBucket") def init_multipart_upload(self, key, headers=None, params=None, upload_context=None): """客户端加密初始化分片上传。 :param params :param upload_context :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :return: :class:`InitMultipartUploadResult ` 返回值中的 `crypto_multipart_context` 记录了加密Meta信息，在upload_part时需要一并传入 """ headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) if not upload_context or not upload_context.data_size: raise ClientError("It is not support none upload_context and must specify data_size of upload_context ") logger.info("Start to init multipart upload by CryptoBucket, data_size: {0}, part_size: {1}".format( upload_context.data_size, upload_context.part_size)) if upload_context.part_size: res = self.crypto_provider.cipher.is_valid_part_size(upload_context.part_size, upload_context.data_size) if not res: raise ClientError("part_size is invalid for multipart upload for CryptoBucket") else: upload_context.part_size = self.crypto_provider.cipher.determine_part_size(upload_context.data_size) content_crypto_material = self.crypto_provider.create_content_material() upload_context.content_crypto_material = content_crypto_material headers = content_crypto_material.to_object_meta(headers, upload_context) resp = super(CryptoBucket, self).init_multipart_upload(key, headers) return resp def upload_part(self, key, upload_id, part_number, data, progress_callback=None, headers=None, upload_context=None): """客户端加密上传一个分片。 :param upload_context: :param str key: 待上传文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 :param str upload_id: 分片上传ID :param int part_number: 分片号，最小值是1. :param data: 待上传数据。 :param progress_callback: 用户指定进度回调函数。可以用来实现进度条等功能。参考 :ref:`progress_callback` 。 :param headers: 用户指定的HTTP头部。可以指定Content-MD5头部等 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :return: :class:`PutObjectResult ` """ logger.info( "Start to upload multipart of CryptoBucket, upload_id = {0}, part_number = {1}".format(upload_id, part_number)) headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) if upload_context: context = upload_context else: raise ClientError("Could not init upload context, upload contexts flag is False and upload context is none") content_crypto_material = context.content_crypto_material if content_crypto_material.cek_alg != self.crypto_provider.cipher.alg or content_crypto_material.wrap_alg != \ self.crypto_provider.wrap_alg: err_msg = 'Envelope or data encryption/decryption algorithm is inconsistent' raise InconsistentError(err_msg, self) headers = content_crypto_material.to_object_meta(headers, context) plain_key = self.crypto_provider.decrypt_encrypted_key(content_crypto_material.encrypted_key) plain_iv = self.crypto_provider.decrypt_encrypted_iv(content_crypto_material.encrypted_iv) offset = context.part_size * (part_number - 1) counter = self.crypto_provider.cipher.calc_offset(offset) cipher = copy.copy(content_crypto_material.cipher) cipher.initialize(plain_key, plain_iv, counter) data = self.crypto_provider.make_encrypt_adapter(data, cipher) resp = super(CryptoBucket, self).upload_part(key, upload_id, part_number, data, progress_callback, headers) return resp def complete_multipart_upload(self, key, upload_id, parts, headers=None): """客户端加密完成分片上传，创建文件。 当所有分片均已上传成功，才可以调用此函数 :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 :param str upload_id: 分片上传ID :param parts: PartInfo列表。PartInfo中的part_number和etag是必填项。其中的etag可以从 :func:`upload_part` 的返回值中得到。 :type parts: list of `PartInfo ` :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :return: :class:`PutObjectResult ` """ logger.info("Start to complete multipart upload of CryptoBucket, upload_id = {0}".format(upload_id)) headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) try: resp = super(CryptoBucket, self).complete_multipart_upload(key, upload_id, parts, headers) except exceptions as e: raise e return resp def abort_multipart_upload(self, key, upload_id, headers=None): """取消分片上传。 :param headers: 可以是dict，建议是oss2.CaseInsensitiveDict :param str key: 待上传的文件名，这个文件名要和 :func:`init_multipart_upload` 的文件名一致。 :param str upload_id: 分片上传ID :return: :class:`RequestResult ` """ logger.info("Start to abort multipart upload of CryptoBucket, upload_id = {0}".format(upload_id)) headers = http.CaseInsensitiveDict(headers) self._init_user_agent(headers) try: resp = super(CryptoBucket, self).abort_multipart_upload(key, upload_id) except exceptions as e: raise e return resp def upload_part_copy(self, source_bucket_name, source_key, byte_range, target_key, target_upload_id, target_part_number, headers=None): """分片拷贝。把一个已有文件的一部分或整体拷贝成目标文件的一个分片。 :param target_part_number: :param target_upload_id: :param target_key: :param source_key: :param source_bucket_name: :param byte_range: 指定待拷贝内容在源文件里的范围。参见 :ref:`byte_range` :param headers: HTTP头部 :type headers: 可以是dict，建议是oss2.CaseInsensitiveDict :return: :class:`PutObjectResult ` """ raise ClientError("The operation is not support for CryptoBucket now") def process_object(self, key, process): raise ClientError("The operation is not support for CryptoBucket")