utils.py 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
  1. # -*- coding: utf-8 -*-
  2. #!/usr/bin/env python
  3. import logging
  4. import os
  5. import sys
  6. import json
  7. from django.conf import settings
  8. from oss2.headers import OSS_OBJECT_TAGGING
  9. from typing import List, Any, Callable, Union, TYPE_CHECKING
  10. from threading import Thread, Timer
  11. from apilib.utils_json import JsonResponse
  12. from apilib.utils_sys import MyStringIO
  13. from apps.thirdparties import AliOSS
  14. from apps.web.core.exceptions import EmptyInput
  15. if TYPE_CHECKING:
  16. from collections import OrderedDict
  17. # sys
  18. PY3 = sys.version_info[0] == 3
  19. if PY3:
  20. unicode_type = str
  21. basestring_type = (str, bytes)
  22. else:
  23. unicode_type = unicode
  24. basestring_type = basestring
  25. logger = logging.getLogger(__name__)
  26. def encode(s):
  27. return s.encode('utf-8') if isinstance(s, unicode_type) else s
  28. def decode(s):
  29. return s.decode('utf-8') if isinstance(s, bytes) else s
  30. class _Missing(object):
  31. def __repr__(self):
  32. return '_Missing'
  33. _missing = _Missing()
  34. MaybePayload = Union[dict, _Missing]
  35. def responder(result, description='', payload=_missing):
  36. # type:(int, unicode, MaybePayload)->JsonResponse
  37. payload = payload if payload is not _missing else {}
  38. return JsonResponse({'result': result, 'description': description, 'payload': payload})
  39. def JsonOkResponse(description='', payload=_missing):
  40. # type:(unicode, MaybePayload)->JsonResponse
  41. return responder(result=1, description=description, payload=payload)
  42. def JsonErrorResponse(description='', payload=_missing):
  43. # type:(unicode, MaybePayload)->JsonResponse
  44. return responder(result=0, description=description, payload=payload)
  45. def NoCommissionErrorResponse(description='', payload=_missing): # type:(unicode, MaybePayload)->JsonResponse
  46. """
  47. result == 2的返回 前端的fail不接管
  48. """
  49. return responder(result=2, description=description, payload=payload)
  50. def DefaultJsonErrorResponse(): return JsonResponse({'result': 0, 'description': u'系统错误', 'payload': {}})
  51. def validate_class_type_arguments(operator):
  52. """
  53. borrowed from maya
  54. Decorator to validate all the arguments to function
  55. are of the type of calling class for passed operator
  56. """
  57. def inner(function):
  58. def wrapper(self, *args, **kwargs):
  59. for arg in args + tuple(kwargs.values()):
  60. if not isinstance(arg, self.__class__):
  61. raise TypeError(
  62. 'unorderable types: {}() {} {}()'.format(
  63. type(self).__name__, operator, type(arg).__name__
  64. )
  65. )
  66. return function(self, *args, **kwargs)
  67. return wrapper
  68. return inner
  69. def validate_arguments_type_of_function(param_type=None):
  70. """
  71. borrowed from maya
  72. Decorator to validate the <type> of arguments in
  73. the calling function are of the `param_type` class.
  74. if `param_type` is None, uses `param_type` as the class where it is used.
  75. Note: Use this decorator on the functions of the class.
  76. """
  77. def inner(function):
  78. def wrapper(self, *args, **kwargs):
  79. type_ = param_type or type(self)
  80. for arg in args + tuple(kwargs.values()):
  81. if not isinstance(arg, type_):
  82. raise TypeError(
  83. (
  84. 'Invalid Type: {}.{}() accepts only the '
  85. 'arguments of type "<{}>"'
  86. ).format(
  87. type(self).__name__,
  88. function.__name__,
  89. type_.__name__,
  90. )
  91. )
  92. return function(self, *args, **kwargs)
  93. return wrapper
  94. return inner
  95. def generate_excel_report(filePath, records, localSave = False):
  96. # type: (str, List[OrderedDict], bool)->None
  97. if (not localSave) and settings.UPLOAD_REPORT_TO_OSS:
  98. with MyStringIO() as fp:
  99. import pandas as pd
  100. df = pd.DataFrame(records)
  101. df.to_excel(fp, sheet_name = 'sheet1', index = False)
  102. fp.seek(0)
  103. AliOSS().put_attachment(key = filePath, data = fp, headers = {
  104. 'Content-Disposition': 'attachment',
  105. OSS_OBJECT_TAGGING: 'lifecycle=7'
  106. })
  107. else:
  108. dest_dir = os.path.join(os.path.dirname(settings.MEDIA_ROOT), filePath)
  109. if not os.path.exists(os.path.dirname(dest_dir)):
  110. os.makedirs(dest_dir)
  111. import pandas as pd
  112. df = pd.DataFrame(records)
  113. df.to_excel(dest_dir, sheet_name = 'sheet1', index = False)
  114. def gernerate_excel_report_for_sheet(filePath, sheets, localSave = False):
  115. if (not localSave) and settings.UPLOAD_REPORT_TO_OSS:
  116. with MyStringIO() as fp:
  117. import pandas as pd
  118. with pd.ExcelWriter(fp) as writer:
  119. for _sheet in sheets:
  120. pd.DataFrame(_sheet.data).to_excel(writer, sheet_name = _sheet.name, index = False)
  121. writer.save()
  122. fp.seek(0)
  123. AliOSS().put_attachment(key = filePath, data = fp.read(), headers = {
  124. 'Content-Disposition': 'attachment',
  125. OSS_OBJECT_TAGGING: 'lifecycle=7'
  126. })
  127. else:
  128. dest_dir = os.path.join(os.path.dirname(settings.MEDIA_ROOT), filePath)
  129. if not os.path.exists(os.path.dirname(dest_dir)):
  130. os.makedirs(dest_dir)
  131. import pandas as pd
  132. with pd.ExcelWriter(dest_dir) as writer:
  133. for _sheet in sheets:
  134. pd.DataFrame(_sheet.data).to_excel(writer, sheet_name = _sheet.name, index = False)
  135. def async_operation(function, *args, **kwargs):
  136. # type:(Callable, *Any, **Any)->None
  137. """
  138. 目前是使用线程,后期考虑引入消息队列或流计算
  139. :param function:
  140. :param args:
  141. :return:
  142. """
  143. logger.debug('to start async thread worker. module = {}, funciton = {}, args = {}, kwargs = {}'.format(
  144. getattr(function, '__module__', None), getattr(function, '__name__', None), str(args), str(kwargs)))
  145. try:
  146. t = Thread(target = function, args = args, kwargs = kwargs)
  147. t.setName('AsyncOperation')
  148. t.setDaemon(False)
  149. t.start()
  150. except Exception as e:
  151. logger.exception(e)
  152. def async_operation_no_catch(function, *args, **kwargs):
  153. # type:(Callable, *Any, **Any)->None
  154. """
  155. 目前是使用线程,后期考虑引入消息队列或流计算
  156. :param function:
  157. :param args:
  158. :return:
  159. """
  160. logger.debug('to start async thread worker. module = {}, function = {}, args = {}, kwargs = {}'.format(
  161. getattr(function, '__module__', None), getattr(function, '__name__', None), str(args), str(kwargs)))
  162. t = Thread(target = function, args = args, kwargs = kwargs)
  163. t.setName('AsyncOperationNoCatch')
  164. t.setDaemon(False)
  165. t.start()
  166. def delay_async_operation(func, delay, *args, **kwargs):
  167. """
  168. 异步延时
  169. :param func:
  170. :param delay:
  171. :param args:
  172. :param kwargs:
  173. :return:
  174. """
  175. logger.debug('to start async thread worker. module = {}, func = {}, args = {}, kwargs = {}'.format(
  176. getattr(func, '__module__', None), getattr(func, '__name__', None), str(args), str(kwargs)))
  177. try:
  178. t = Timer(interval=delay, function=func, args=args, kwargs=kwargs)
  179. t.setName('DelayAsyncOperation')
  180. t.setDaemon(False)
  181. t.start()
  182. except Exception as e:
  183. logger.exception(e)
  184. def parse_json_payload(string, allow_empty=True):
  185. # type:(str, bool)->dict
  186. if not string and allow_empty:
  187. return {}
  188. elif not string and not allow_empty:
  189. raise EmptyInput('empty input is not allowed')
  190. else:
  191. return json.loads(string)