manyi/script/ad/build_ad_aggregates.py

# -*- coding: utf-8 -*-
#!/usr/bin/env python
import datetime

import click
from script.base import init_env

init_env(interactive=True)

import simplejson as json
import redis

from apps.web.agent.models import Agent
from apps.web.dealer.models import Dealer
from apps.web.ad.models import AdRecord, Advertisement
from apps.web.core.db import Query
from apps.web.constant import Const

redis_client = redis.Redis(host='121.41.75.233', port=6379, db=0)

class NotSupported(Exception): pass

@click.group()
def cli():
    click.echo(u'build ad aggregate')

def _build_query(category, start_date, end_date):
    if not category or category not in ('ad', 'dealer', 'agent', 'all'): raise NotSupported('only support ad or dealer or agent or all')

    click.echo('building query for %s' % (category,))

    pre_records = []
    rsd_manager_id = '5abc5f5c4864d0265c654cb0'

    if category == 'dealer':

        agents =  [ str(_.id) for _ in Agent.objects(managerId=rsd_manager_id)]
        pre_records = [ {'dealerId': str(_.id), 'agentId': str(_.agentId)} for _ in Dealer.objects(agentId__in=agents) ]

    elif category == 'ad':
        pre_records = [ {'adId': str(_.adId)} for _ in Advertisement.objects(managerId=rsd_manager_id) ]

    elif category == 'agent':
        pre_records =  [ {'agentId' :str(_.id)} for _ in Agent.objects(managerId=rsd_manager_id)]

    elif category == 'all':

        agents = [str(_.id) for _ in Agent.objects(managerId=rsd_manager_id)]
        pre_records =   [{'dealerId': str(_.id), 'agentId': str(_.agentId)} for _ in Dealer.objects(agentId__in=agents)] \
                        + [{'adId': str(_.adId)} for _ in Advertisement.objects(managerId=rsd_manager_id)] \
                        + [{'agentId': str(_.id)} for _ in Agent.objects(managerId=rsd_manager_id)]

    l = []
    for _ in pre_records:
        attrs = _.copy()

        startTime = datetime.datetime.strptime(start_date, Const.DATE_FMT).replace(hour=0, minute=0, second=0, microsecond=0)
        endTime = datetime.datetime.strptime(end_date, Const.DATE_FMT).replace(hour=23, minute=59, second=59, microsecond=999999)

        attrs['dateTimeAdded__gte'] = startTime
        attrs['dateTimeAdded__lte'] = endTime

        _['startTime'] = start_date
        _['endTime'] = end_date


        l.append(Query(raw=_, attrs=attrs))

    click.echo('%d queries built' % (len(l),))
    click.echo('\n'.join([_.to_string() for _ in l]))

    return l

@cli.command()
@click.option('--category', '-c', prompt=u'为广告生成还是为经销商生成(ad, dealer, agent)')
@click.option('--start_date', '-s', prompt=u'起始日期')
@click.option('--end_date', '-e', prompt=u'终止日期')
def build_query(category, start_date, end_date):
    _build_query(category, start_date, end_date)


@cli.command()
@click.option('--category', '-c', prompt=u'为广告生成还是为经销商生成(ad, dealer, agent)')
@click.option('--start_date', '-s', prompt=u'起始日期')
@click.option('--end_date', '-e', prompt=u'终止日期')
def build_reports(category, start_date, end_date):
    """
    为了加快广告效果统计的展现，考虑加入增量报表
    考虑加入的key
    (managerId,), (managerId, agentId), (managerId, agentId, dealerId)...
    :return:
    """
    queries = _build_query(category, start_date, end_date)
    for q in queries:
        objects = AdRecord.objects(converted=True, **q.attrs).timeout(False)
        _ = [_.to_dict_in_cn() for _ in objects]
        print 'processing', q.to_string()
        redis_client.setnx(q.hashed, json.dumps(_))
        del objects
        del _


if __name__ == '__main__':
    cli()