manyi/script/spider/yunyichong.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os, sys,time,datetime
import urllib
import requests
from mongoengine import register_connection, PointField, DynamicDocument, StringField
import simplejson as json

from django.db.models.fields import DateTimeField

PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")

from script.base import init_env

init_env(interactive = False)

from apps.web.core.db import Searchable

register_connection(alias = 'spider',
                    name = 'spider',
                    host = '116.62.228.194',
                    port = 27017,
                    username = 'dba',
                    password = 'dayuan@2020..',
                    authentication_source = 'admin')

class Goverment(Searchable):
    province = StringField(default = "")
    city = StringField(default = "")
    name = StringField(default = '')
    lat = StringField(default = '')
    lng = StringField(default = '') 
    
    meta = {
        'collection': 'Goverment',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
    
class yunyichongDevice(Searchable):
    uuid = StringField(default = '')
    meta = {
        'collection': 'yunyichong_device',
        'db_alias': 'spider',
        'unique_together': {'uuid'}
    }

class errorLog(Searchable):
    meta = {
        'collection': 'wanzhuang_err_location',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
# 根据二维码编号，获取设备更详细的信息
url = 'https://wx-yyc.dianxiaomei.com.cn/ui/nearby?lat=%s&lng=%s&city='
cookie1 = 'PHPSESSID=0vcenbi6pgu6stup1brlelmo42; yyc_auth_v3=HNfsUaXyxVUL68umVR23r1JDNJd%2BwEEMXCH1oSMEbg%2FE08uwJGjseQE%2F7QigXiBx; uid=19599418;'
govs = Goverment.get_collection().find()
ii = 0
for gov in govs[19970:]:
    try:
        ii += 1
        print ii,gov['name']
        if gov['name'][-2:] != u'政府':
            continue 
        url1 = url % (gov['lat'],gov['lng'])
        cookies = cookie1 + '_lat=%s; _lng=%s;' % (gov['lat'],gov['lng']) + 'j_w_city=%E6%AD%A6%E6%B1%89%E5%B8%82'
        strhtml = str(requests.get(url1,verify=False,headers = {'Cookie':cookies}).text)
        startIndex = strhtml.find('device:[')
        endIndex = strhtml.find(']',startIndex) 
        if not startIndex or not endIndex:
            continue
        devListStr = strhtml[startIndex+7:endIndex+1]
        devListDict = json.loads('{"device":%s}' % devListStr)
        
        for dev in devListDict['device']:
            yunyichongDevice.get_collection().update({'uuid':dev['uuid']},{'$set':dev},upsert = True)
    except Exception,e:
        continue
print('OK')