manyi/script/db/remove_service_progress.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import datetime
import os
import sys
import time

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'configs.production')

PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

from script.base import init_env, get_logger

init_env(interactive = False)

logger = get_logger(__name__)

from apilib.utils_mongo import BulkHandlerEx
from apps.web.constant import Const
from apps.web.user.models import ServiceProgress


def remove_serviceProgress_periodically():
    threeDaysAgo = datetime.datetime.now() - datetime.timedelta(days = 3)
    threeDaysAgo = int(time.mktime(threeDaysAgo.timetuple()))

    threeMonthAgo = datetime.datetime.now() - datetime.timedelta(days = 91)
    threeMonthAgo = int(time.mktime(threeMonthAgo.timetuple()))

    start_time = int(time.time())

    while True:
        now_time = int(time.time())
        if now_time - start_time > 2 * 60 * 60:
            logger.debug('time is over. wait next.')
            return

        items = ServiceProgress.get_collection().find(
            {
                '$or': [{'finished_time': {'$lt': threeDaysAgo}}, {'isFinished': True}],
                'devTypeCode': {
                    '$ne': Const.DEVICE_TYPE_CODE_HP_GATE
                }}, {'_id': 1}).limit(2000)

        bulker = BulkHandlerEx(ServiceProgress.get_collection())

        for item in items:
            bulker.delete(query_dict = {'_id': item['_id']})

        count = len(bulker.requests)

        if count > 0:
            logger.debug('prepare to delete {} rows.'.format(count))

            bulker.execute()
            bulker = BulkHandlerEx(ServiceProgress.get_collection())  # type: BulkHandlerEx

        if count < 2000:
            break

    while True:
        now_time = int(time.time())
        if now_time - start_time > 2 * 60 * 60:
            logger.debug('time is over. wait next.')
            return

        items = ServiceProgress.get_collection().find(
            {
                '$or': [
                    {'finished_time': {'$lt': threeMonthAgo}},
                    {'isFinished': True}
                ],
                'finished_time': {'$lt': threeDaysAgo},
                'devTypeCode': Const.DEVICE_TYPE_CODE_HP_GATE
            }, {'_id': 1}).limit(2000)

        bulker = BulkHandlerEx(ServiceProgress.get_collection())  # type: BulkHandlerEx

        for item in items:
            bulker.delete(query_dict = {'_id': item['_id']})

        count = len(bulker.requests)

        if count > 0:
            logger.debug('prepare to delete {} rows.'.format(count))

            bulker.execute()
            bulker = BulkHandlerEx(ServiceProgress.get_collection())  # type: BulkHandlerEx

        if count < 2000:
            break

    logger.debug('delete all over.')


remove_serviceProgress_periodically()