# -*- coding: utf-8 -*- # !/usr/bin/env python import os, sys,time,datetime import urllib import requests from mongoengine import register_connection, PointField, DynamicDocument, StringField import simplejson as json from django.db.models.fields import DateTimeField PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..') sys.path.insert(0, PROJECT_ROOT) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing") from script.base import init_env init_env(interactive = False) from apps.web.core.db import Searchable register_connection(alias = 'spider', name = 'spider', host = '116.62.228.194', port = 27017, username = 'dba', password = 'dayuan@2020..', authentication_source = 'admin') class yunxiangDevice(Searchable): meta = { 'collection': 'yunxiang_device', 'db_alias': 'spider', 'unique_together': {'gtel'} } # 先把所有的设备二维码编号爬下来 # page=0 # while True: # url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page # strhtml = requests.get(url,timeout = 15).text # result = json.loads(strhtml) # if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0: # break # for dev in result['data']['content']: # xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True) # page += 1 # 根据二维码编号,获取设备更详细的信息 url1 = 'http://cdz.gpsserver.cn/ChargeCar?gtel=680000%05d' for ii in range(20000): try: print ii url = url1 % ii strhtml = requests.get(url,timeout = 15).text result = json.loads(strhtml) if len(result) == 0: continue data = result[0] yunxiangDevice.get_collection().update({'gtel':data['gtel']},{'$set':data},upsert=True) except Exception,e: continue print('OK')