# -*- coding: utf-8 -*- # !/usr/bin/env python import os, sys,time,datetime import urllib import requests from mongoengine import register_connection, PointField, DynamicDocument, StringField import simplejson as json from django.db.models.fields import DateTimeField PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..') sys.path.insert(0, PROJECT_ROOT) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing") from script.base import init_env init_env(interactive = False) from apps.web.core.db import Searchable register_connection(alias = 'spider', name = 'spider', host = '211.159.224.10', port = 27119, username = 'service', password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5', authentication_source = 'admin') class Goverment(Searchable): province = StringField(default = "") city = StringField(default = "") name = StringField(default = '') lat = StringField(default = '') lng = StringField(default = '') meta = { 'collection': 'Goverment', 'db_alias': 'spider', 'unique_together': {'lat', 'lng'} } class wanzhuangDevice(Searchable): lat = StringField(default = '') lng = StringField(default = '') meta = { 'collection': 'wanzhuang_device', 'db_alias': 'spider', 'unique_together': {'device_num'} } class errorLog(Searchable): meta = { 'collection': 'wanzhuang_err_location', 'db_alias': 'spider', 'unique_together': {'lat', 'lng'} } # 根据二维码编号,获取设备更详细的信息 url = 'https://webapp.wanzhuangkj.com/api/record/deviceMap?longitude=%s&latitude=%s&scale=50000&device_type=-1' auth = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJpc3MiOiJodHRwOi8vYXBpLndlYmFwcC53YW56aHVhbmdrai5jb20vYXV0aC90b2tlbiIsImlhdCI6MTYxNTE4MjUxNiwiZXhwIjoxNjE2MzkyMTE2LCJuYmYiOjE2MTUxODI1MTYsImp0aSI6ImpPb2RLamw0ZXhmdUQwajEiLCJzdWIiOjE4NTAzMjIsInBydiI6IjZmMTdkOTMxYmZlNjUwYzNlMjU3YWNkNGE5NDMxYWU5ZWM1MDEyNjciLCJjb21wYW55X2lkIjoiMiIsImNvbXBhbnlEYl9jb25uIjoid3oiLCJ1c2VyX2lkIjoxNzg4MDY1LCJ1c2VyX3R5cGUiOiJ3eHBheSIsIm9wZW5pZCI6Im9feGdxMHF6eUJPVHpQbmNRWU1EVWt5RFoyeVEifQ.2bPZFgPJPS6O5N0_XKXjLKsUI57d-QiHTClpjUWUpvs' govs = Goverment.get_collection().find() ii = 0 for gov in govs: try: ii += 1 print ii,gov['name'] url1 = url % (gov['lng'],gov['lat']) strhtml = requests.get(url1,verify=False,headers = {'Authorization':auth}).text result = json.loads(strhtml) if result.has_key('status_code') and result['status_code'] == 429: time.sleep(20) errorLog.get_collection().update({'lat':gov['lat'],'lng':gov['lng']}) if not result.has_key('data'): continue if len(result['data'])== 0: continue print len(result['data']) for dev in result['data']: wanzhuangDevice.get_collection().update({'device_num':dev['device_num']},{'$set':dev},upsert = True) except Exception,e: continue print('OK')