manyi/script/spider/pianyichong.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os, sys,time,datetime
import urllib
import requests
from mongoengine import register_connection, PointField, DynamicDocument, StringField
import simplejson as json

from django.db.models.fields import DateTimeField

PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")

from script.base import init_env

init_env(interactive = False)

from apps.web.core.db import Searchable

register_connection(alias = 'spider',
                    name = 'spider',
                    host = '211.159.224.10',
                    port = 27119,
                    username = 'service',
                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
                    authentication_source = 'admin')

class Goverment(Searchable):
    province = StringField(default = "")
    city = StringField(default = "")
    name = StringField(default = '')
    lat = StringField(default = '')
    lng = StringField(default = '') 
    
    meta = {
        'collection': 'Goverment',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
    
class pianyichongDevice(Searchable):
    meta = {
        'collection': 'pianyichong_device',
        'db_alias': 'spider',
        'unique_together': {'id'}
    }

# 根据二维码编号，获取设备更详细的信息
url1 = 'http://www.bianyichong.cn/prod/prodInstlPos/queryGeo'
ii = 43570
govs = Goverment.get_collection().find()

devIdMap = {}
for dev in pianyichongDevice.get_collection().find({}):
    devIdMap[dev['id']] = 1

for gov in govs[ii:]:
    try:
        ii += 1
        print ii
        if not (u'政府' in gov['name'] or u'局' in gov['name']):
            continue
        
        data = {'lng':gov['lng'],'lat':gov['lat'],'bizNo':'B100000','distance':3000}
        strhtml = requests.post(url1,data=data).text
        result = json.loads(strhtml)
        if not result.has_key('model'):
            continue
        if not result['model'].has_key('prodInstlPosList') or len(result['model']['prodInstlPosList']) == 0:
            continue
        for dev in result['model']['prodInstlPosList']:
            if dev['id'] in devIdMap:
                continue
            pianyichongDevice.get_collection().update({'id':dev['id']},{'$set':dev},upsert = True)
            devIdMap[dev['id']] = 1
            print 'new one!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
    except Exception,e:
        print 'eeeeeeeeeeeeeeeeeeeeeee',e
        continue
print('OK')