# -*- coding: utf-8 -*- # !/usr/bin/env python import os, sys,time,datetime import urllib import requests from mongoengine import register_connection, PointField, DynamicDocument, StringField import simplejson as json from django.db.models.fields import DateTimeField PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..') sys.path.insert(0, PROJECT_ROOT) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing") from script.base import init_env init_env(interactive = False) from apps.web.core.db import Searchable register_connection(alias = 'spider', name = 'spider', host = '116.62.228.194', port = 27017, username = 'dba', password = 'dayuan@2020..', authentication_source = 'admin') class Goverment(Searchable): province = StringField(default = "") city = StringField(default = "") name = StringField(default = '') lat = StringField(default = '') lng = StringField(default = '') meta = { 'collection': 'Goverment', 'db_alias': 'spider', 'unique_together': {'lat', 'lng'} } class yunyichongDevice(Searchable): uuid = StringField(default = '') meta = { 'collection': 'yunyichong_device', 'db_alias': 'spider', 'unique_together': {'uuid'} } class errorLog(Searchable): meta = { 'collection': 'wanzhuang_err_location', 'db_alias': 'spider', 'unique_together': {'lat', 'lng'} } # 根据二维码编号,获取设备更详细的信息 url = 'https://wx-yyc.dianxiaomei.com.cn/ui/nearby?lat=%s&lng=%s&city=' cookie1 = 'PHPSESSID=0vcenbi6pgu6stup1brlelmo42; yyc_auth_v3=HNfsUaXyxVUL68umVR23r1JDNJd%2BwEEMXCH1oSMEbg%2FE08uwJGjseQE%2F7QigXiBx; uid=19599418;' govs = Goverment.get_collection().find() ii = 0 for gov in govs[19970:]: try: ii += 1 print ii,gov['name'] if gov['name'][-2:] != u'政府': continue url1 = url % (gov['lat'],gov['lng']) cookies = cookie1 + '_lat=%s; _lng=%s;' % (gov['lat'],gov['lng']) + 'j_w_city=%E6%AD%A6%E6%B1%89%E5%B8%82' strhtml = str(requests.get(url1,verify=False,headers = {'Cookie':cookies}).text) startIndex = strhtml.find('device:[') endIndex = strhtml.find(']',startIndex) if not startIndex or not endIndex: continue devListStr = strhtml[startIndex+7:endIndex+1] devListDict = json.loads('{"device":%s}' % devListStr) for dev in devListDict['device']: yunyichongDevice.get_collection().update({'uuid':dev['uuid']},{'$set':dev},upsert = True) except Exception,e: continue print('OK')