manyi/taskmanager/config.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python
"""
celery的配置文件
apps.tasks.config
~~~~~~~~~~~~~~~~~
"""
import os
from os.path import join, abspath

from dotenv import get_key, load_dotenv
from celery.schedules import crontab


base_dir = os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/..")
task_dir = join(base_dir, 'taskmanager')

dotenv_path = abspath(join(base_dir, '.env.%s' % os.environ.get('MY_ENV')))

load_dotenv(dotenv_path)


def env(key):
    return get_key(dotenv_path, key)


# ========== broker settings ===========
#: broker url
BROKER_URL = env('CELERY_BROKER_URL')

# : 指定接受的内容类型
CELERY_ACCEPT_CONTENT = ['json', 'msgpack']

# ========== celery settings ===========

MY_CELERY_QUEUE = "manyi_celery"
MY_CELERY_ROUTING_KEY = "manyi_celery"

POLL_RECHARGE_RECORD_QUEUE = 'manyi_poll_recharge_record'
POLL_RECHARGE_RECORD_QUEUE_KEY = 'manyi_poll_recharge_record'

DEALER_DATA_ANYLYSE_QUEUE = 'manyi_dealer_data_anylyse'
DEALER_DATA_ANYLYSE_QUEUE_KEY = 'manyi_dealer_data_anylyse'

# 设备离线的延时任务 安全起见这个地方仅仅区里发送任务的通知 不要访问数据库 作为独立的单元处理
DEVICE_OFFLINE_QUEUE = "manyi_device_offline"
DEVICE_OFFLINE_QUEUE_KEY = "manyi_device_offline"

OFFLINE_TASK_QUEUE = 'manyi_offline_task'
OFFLINE_TASK_QUEUE_KEY = 'manyi_offline_task'

REPORT_TASK_QUEUE = 'manyi_report_task'
REPORT_TASK_QUEUE_KEY = 'manyi_report_task'

CELERY_QUEUES = {
    MY_CELERY_ROUTING_KEY: {
        'exchange': MY_CELERY_QUEUE,
        'exchange_type': 'direct',
        'binding_key': MY_CELERY_ROUTING_KEY
    },

    POLL_RECHARGE_RECORD_QUEUE: {
        'exchange': POLL_RECHARGE_RECORD_QUEUE,
        'exchange_type': 'direct',
        'binding_key': POLL_RECHARGE_RECORD_QUEUE_KEY
    },

    DEALER_DATA_ANYLYSE_QUEUE: {
        'exchange': DEALER_DATA_ANYLYSE_QUEUE,
        'exchange_type': 'direct',
        'binding_key': DEALER_DATA_ANYLYSE_QUEUE_KEY
    },

    DEVICE_OFFLINE_QUEUE: {
        'exchange': DEVICE_OFFLINE_QUEUE,
        'exchange_type': 'direct',
        'binding_key': DEVICE_OFFLINE_QUEUE_KEY
    },

    OFFLINE_TASK_QUEUE: {
        'exchange': OFFLINE_TASK_QUEUE,
        'exchange_type': 'direct',
        'binding_key': OFFLINE_TASK_QUEUE_KEY
    },

    REPORT_TASK_QUEUE: {
        'exchange': REPORT_TASK_QUEUE,
        'exchange_type': 'direct',
        'binding_key': REPORT_TASK_QUEUE_KEY
    }
}

CELERY_ROUTES = {
    'tasks.device_offline_notify': {
        'queue': DEVICE_OFFLINE_QUEUE,
        'routing_key': DEVICE_OFFLINE_QUEUE_KEY
    },

    'tasks.report_new_payment_to_dealer_via_wechat': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.report_service_complete_to_user_via_wechat': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.add': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.report_to_user_via_wechat': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.send_msg_to_user_via_wechat': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.report_to_dealer_via_wechat': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.report_feedback_to_dealer_via_wechat': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.send_topic_command': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.send_to_xf_falut': {
        'queue': MY_CELERY_QUEUE,
        'routing_key': MY_CELERY_ROUTING_KEY
    },

    'tasks.generate_simCharge_excel_report': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.generate_dealerWithDraw_excel_report': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.generate_biz_stats_for_manager': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.generate_business_stats_report_by_dealer': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_charge_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_consume_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_send_coins_to_card_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_on_points_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_API_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_group_stat_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_vcard_info_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.export_aggregate_dealer_income': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.import_simcard_excel_to_db': {
        'queue': OFFLINE_TASK_QUEUE,
        'routing_key': OFFLINE_TASK_QUEUE_KEY
    },

    'tasks.export_simcard_excel_from_db': {
        'queue': OFFLINE_TASK_QUEUE,
        'routing_key': OFFLINE_TASK_QUEUE_KEY
    },

    'tasks.export_device_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.poll_dealer_recharge_record': {
        'queue': POLL_RECHARGE_RECORD_QUEUE,
        'routing_key': POLL_RECHARGE_RECORD_QUEUE_KEY
    },

    'tasks.poll_user_recharge_record': {
        'queue': POLL_RECHARGE_RECORD_QUEUE,
        'routing_key': POLL_RECHARGE_RECORD_QUEUE_KEY
    },

    'tasks.manager_export_charge_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.manager_export_dealer_info_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.manager_export_consume_order_excel_from_db': {
        'queue': REPORT_TASK_QUEUE,
        'routing_key': REPORT_TASK_QUEUE_KEY
    },

    'tasks.batch_set_device_params': {
        'queue': OFFLINE_TASK_QUEUE,
        'routing_key': OFFLINE_TASK_QUEUE_KEY
    },

    'tasks.set_device_params': {
        'queue': OFFLINE_TASK_QUEUE,
        'routing_key': OFFLINE_TASK_QUEUE_KEY
    },

    'tasks.batch_set_server_settings': {
        'queue': OFFLINE_TASK_QUEUE,
        'routing_key': OFFLINE_TASK_QUEUE_KEY
    },

    'tasks.set_server_settings': {
        'queue': OFFLINE_TASK_QUEUE,
        'routing_key': OFFLINE_TASK_QUEUE_KEY
    },
}

CELERYBEAT_SCHEDULE = {
    'remove_serviceProgress_periodically': {
        'task': 'tasks.remove_serviceProgress_periodically',
        'schedule': crontab(minute=0, hour=2),
        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    'handle_customer_complaints_yesterday': {
        'task': 'tasks.handle_customer_complaints_yesterday',
        'schedule': crontab(minute=0, hour=3),
        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    'send_SIM_expired_messages_by_sms': {
        'task': 'tasks.send_SIM_expired_messages',
        'schedule': crontab(day_of_month='1,6,11,19,23,25,26', minute = 0, hour = 10),
        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        },
        'args': ('sms',)
    },

    'send_SIM_expired_messages_by_wechat': {
        'task': 'tasks.send_SIM_expired_messages',

        'schedule': crontab(day_of_month = '1,6,11,19,23,25,26', minute = 0, hour = 20),

        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        },

        'args': ('wechat',)
    },

    'calc_dealer_user_count': {
        'task': 'tasks.calc_dealer_user_count',

        'schedule': crontab(minute = 30, hour = 0, day_of_week = 'fri'),

        'options': {
            'queue': DEALER_DATA_ANYLYSE_QUEUE,
            'routing_key': DEALER_DATA_ANYLYSE_QUEUE_KEY
        }
    },

    'generate_manager_map_options': {
        'task': 'tasks.generate_manager_map_options',

        'schedule': crontab(minute = 0, hour = 1),

        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    'check_withdraw_via_bank': {
        'task': 'tasks.check_withdraw_via_bank',

        'schedule': crontab(minute = 0, hour = 2),

        'options': {
            'queue': POLL_RECHARGE_RECORD_QUEUE,
            'routing_key': POLL_RECHARGE_RECORD_QUEUE_KEY
        }
    },

    'check_and_retry_withdraw': {
        'task': 'tasks.check_and_retry_withdraw',

        'schedule': crontab(minute = 0, hour = 4),

        'options': {
            'queue': POLL_RECHARGE_RECORD_QUEUE,
            'routing_key': POLL_RECHARGE_RECORD_QUEUE_KEY
        }
    },

    # 每日晚上凌晨2点生成订单
    "gen_daily_rent_order": {
        "task": "tasks.gen_daily_rent_order",

        "schedule": crontab(minute=0, hour=2),
        "options": {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    # 每日下午3点进行订单扣款
    "deduct_rent_order": {
        "task": "tasks.deduct_rent_order",

        "schedule": crontab(minute=0, hour=15),
        "options": {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    # 每日统计
    "ledger_consume_order_stats": {
        "task": "tasks.ledger_consume_order_stats",

        "schedule": crontab(hour=1, minute=0),
        "options": {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },


    # 每天凌晨 1 点 对于退款订单进行一次拉取信息
    'pull_refund_order': {
        'task': 'tasks.pull_refund_order',
        'schedule': crontab(minute=0, hour=1),

        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    # 每天凌晨 2 点 运行一次经销商的自动提现
    'dealer_auto_withdraw': {
        'task': 'tasks.dealer_auto_withdraw',
        # 每个月1号执行任务
        'schedule': crontab(minute=0, hour=2),

        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    'sum_customer': {
        'task': 'tasks.sum_customer',
        # 每个月1号执行任务
        'schedule': crontab(minute = 0, hour = 0, day_of_month = 1),

        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },

    'make_rpt_into_db': {
        'task': 'tasks.make_rpt_into_db',

        'schedule': crontab(minute = 30, hour = 2),

        'options': {
            'queue': MY_CELERY_QUEUE,
            'routing_key': MY_CELERY_ROUTING_KEY
        }
    },
}

CELERY_ENABLE_UTC = True
CELERY_TIMEZONE = "Asia/Shanghai"

CELERY_IGNORE_RESULT = True

# : 任务序列化和反序列化使用msgpack方案
CELERY_TASK_SERIALIZER = 'msgpack'

# : 读取任务结果一般性能要求不高，所以使用了可读性更好的JSON
CELERY_RESULT_SERIALIZER = 'json'

# Decides if publishing task messages will be retried in the case of connection loss or other connection errors
CELERY_TASK_PUBLISH_RETRY = False

# ：每个worker处理的最大任务数量 超过任务数量直接销毁， 同时释放内存
CELERYD_MAX_TASKS_PER_CHILD = 100

# : 任务过期时间，不建议直接写86400，应该让这样的magic数字表述更明显
CELERY_TASK_RESULT_EXPIRES = 60 * 60 * 24

#: 结果储存
CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND')

CELERYD_LOG_FORMAT = '[%(asctime)s] %(levelname)s[%(processName)s] %(module)s [%(name)s:%(lineno)d]: %(message)s'

CELERYD_TASK_LOG_FORMAT = '[%(asctime)s] %(levelname)s[%(processName)s] %(module)s [%(name)s:%(lineno)d]:' \
                          ' [%(task_name)s(%(task_id)s)] %(message)s'