manyi/script/spider/wanzhuang.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os, sys,time,datetime
import urllib
import requests
from mongoengine import register_connection, PointField, DynamicDocument, StringField
import simplejson as json

from django.db.models.fields import DateTimeField

PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")

from script.base import init_env

init_env(interactive = False)

from apps.web.core.db import Searchable

register_connection(alias = 'spider',
                    name = 'spider',
                    host = '211.159.224.10',
                    port = 27119,
                    username = 'service',
                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
                    authentication_source = 'admin')

class Goverment(Searchable):
    province = StringField(default = "")
    city = StringField(default = "")
    name = StringField(default = '')
    lat = StringField(default = '')
    lng = StringField(default = '') 
    
    meta = {
        'collection': 'Goverment',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
    
class wanzhuangDevice(Searchable):
    lat = StringField(default = '')
    lng = StringField(default = '') 
    meta = {
        'collection': 'wanzhuang_device',
        'db_alias': 'spider',
        'unique_together': {'device_num'}
    }

class errorLog(Searchable):
    meta = {
        'collection': 'wanzhuang_err_location',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
# 根据二维码编号，获取设备更详细的信息
url = 'https://webapp.wanzhuangkj.com/api/record/deviceMap?longitude=%s&latitude=%s&scale=50000&device_type=-1'
auth = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwOi8vYXBpLndlYmFwcC53YW56aHVhbmdrai5jb20vYXV0aC90b2tlbiIsImlhdCI6MTYxNTE4MjUxNiwiZXhwIjoxNjE2MzkyMTE2LCJuYmYiOjE2MTUxODI1MTYsImp0aSI6ImpPb2RLamw0ZXhmdUQwajEiLCJzdWIiOjE4NTAzMjIsInBydiI6IjZmMTdkOTMxYmZlNjUwYzNlMjU3YWNkNGE5NDMxYWU5ZWM1MDEyNjciLCJjb21wYW55X2lkIjoiMiIsImNvbXBhbnlEYl9jb25uIjoid3oiLCJ1c2VyX2lkIjoxNzg4MDY1LCJ1c2VyX3R5cGUiOiJ3eHBheSIsIm9wZW5pZCI6Im9feGdxMHF6eUJPVHpQbmNRWU1EVWt5RFoyeVEifQ.2bPZFgPJPS6O5N0_XKXjLKsUI57d-QiHTClpjUWUpvs'
govs = Goverment.get_collection().find()
ii = 0
for gov in govs:
    try:
        ii += 1
        print ii,gov['name']
        url1 = url % (gov['lng'],gov['lat'])
        strhtml = requests.get(url1,verify=False,headers = {'Authorization':auth}).text
        result = json.loads(strhtml)
        if result.has_key('status_code') and result['status_code'] == 429:
            time.sleep(20)
            errorLog.get_collection().update({'lat':gov['lat'],'lng':gov['lng']})
        if not result.has_key('data'):
            continue
        if len(result['data'])== 0:
            continue
        print len(result['data'])
        for dev in result['data']:
            wanzhuangDevice.get_collection().update({'device_num':dev['device_num']},{'$set':dev},upsert = True)
    except Exception,e:
        continue
print('OK')