manyi/script/spider/shankailaidian.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os, sys,time,datetime
import urllib
import requests
from mongoengine import register_connection, PointField, DynamicDocument, StringField
import simplejson as json

from django.db.models.fields import DateTimeField

PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")

from script.base import init_env

init_env(interactive = False)

from apps.web.core.db import Searchable

register_connection(alias = 'spider',
                    name = 'spider',
                    host = '211.159.224.10',
                    port = 27119,
                    username = 'service',
                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
                    authentication_source = 'admin')

class Goverment(Searchable):
    province = StringField(default = "")
    city = StringField(default = "")
    name = StringField(default = '')
    lat = StringField(default = '')
    lng = StringField(default = '') 
    
    meta = {
        'collection': 'Goverment',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
    
class shankailaidianDevice(Searchable):
    meta = {
        'collection': 'shankailaidian_device',
        'db_alias': 'spider',
        'unique_together': {'vStationNo'}
    }

# 根据二维码编号，获取设备更详细的信息
url1 = 'https://api.issks.com/issksapi/V2/ec/stationList/json.shtml?mapX=%s&mapY=%s'
ii = 0
govs = Goverment.get_collection().find({})

devIdMap = {}
for dev in shankailaidianDevice.get_collection().find({}):
    devIdMap[dev['vStationNo']] = 1

cookie1 = 'SECKEY_ABVK=9Evaq2OKVzgoe1gStw9X4D1UBWkr2l7/0NRT+NUeCJM%3D; BMAP_SECKEY=zkVWrAyscY50tP8ct7mfV8gd7WUVrYi84X5_k1PtFRpswZA8LXI2sUKNQ0c8MUI2xqDaQuWJa5VqB3OiOZNn0zokd5VNvfOykp0gHv46O1UvEecnTCvXqGcz8Qu6CZMA6NcAEK-wGEIW8wIV2H1c4-Iq5avaGxXajjaUVgVgm1NvCcAaN9PAaz4276H_pE7d; JSESSIONID=DDA5CAD77862B1B692D5055FD8F4818B; 14a03328ed59437f0a5a7fea4391b872=3b935069ae8453531d5ebb3e151b6cac'
agent = 'Mozilla/5.0 (Linux; Android 7.1.2; HLTE212T Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/86.0.4240.99 XWEB/3170 MMWEBSDK/201101 Mobile Safari/537.36 MMWEBID/6505 MicroMessenger/7.0.21.1800(0x2700157F) Process/toolsmp WeChat/arm32 Weixin NetType/WIFI Language/zh_CN ABI/arm64'
refer = 'https://api.issks.com/issksapi/V2/ec/stationList.shtml'

for gov in govs[ii:]:
    try:
        ii += 1
        print gov['province'],gov['city'],gov['name'],ii
        if not (u'政府' in gov['name']):
            continue
        
        url = url1 % (gov['lng'],gov['lat'])
        strhtml = requests.get(url,headers = {'Cookie':cookie1,'User-Agent':agent,'Refer':refer}).text
        result = json.loads(strhtml)
        if not result.has_key('list'):
            continue
        
        for dev in result['list']:
            if dev['vStationNo'] in devIdMap:
                continue
            shankailaidianDevice.get_collection().update({'vStationNo':dev['vStationNo']},{'$set':dev},upsert = True)
            devIdMap[dev['vStationNo']] = 1
            print 'new one!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
    except Exception,e:
        print 'eeeeeeeeeeeeeeeeeeeeeee',e
        continue
print('OK')