manyi/script/spider/weitiandi.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os, sys,time,datetime
import urllib
import requests
from mongoengine import register_connection, PointField, DynamicDocument, StringField
import simplejson as json

from django.db.models.fields import DateTimeField

PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")

from script.base import init_env

init_env(interactive = False)

from apps.web.core.db import Searchable

register_connection(alias = 'spider',
                    name = 'spider',
                    host = '211.159.224.10',
                    port = 27119,
                    username = 'service',
                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
                    authentication_source = 'admin')

class weitiandiDevice(Searchable):
    meta = {
        'collection': 'weitiandi_device',
        'db_alias': 'spider',
    }
    
# 根据二维码编号，获取设备更详细的信息
logicalFormat1 = 'XC%04d'
logicalFormat2 = 'XC%05d'

url1 = 'http://wetiandi.com/app/index.php?i=1&j=1&c=entry&m=wdl_shopping&do=wepay&sn=GD1B%s'
cookie1 = 'acw_tc=2f624a4616505087021958680e4bd61e40de02b46746403ae5406a5920172a; PHPSESSID=289df6867a76371820780cdbedce0397; PHPSESSID=289df6867a76371820780cdbedce0397'
agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63060012)'
#36000-80000
newCount = 0
for ii in range(20000,110000): 
    try:
        sn = 'GD1B%s' % ii
        if weitiandiDevice.get_collection().find({'sn':sn}).count() >= 1:
            print 'exist one',ii
            continue
        url = url1 % ii
        response = requests.get(url,headers = {'Cookie':cookie1,'User-Agent':agent})
        if '不存在' in response.content or '抱歉' in response.content or '异常' in response.content:
            print 'offline',ii
            continue
        weiidIndex = response.content.find('var dev = ')
        if not weiidIndex:
            continue
        endIndex = response.content.find(';',weiidIndex+11)
        if not endIndex:
            continue
        devInfo = response.content[weiidIndex+10:endIndex]
        devDict = json.loads(devInfo)
        newCount += 1
        print 'catch new one !',newCount
        
        devDict['spiderData'] = '0421'
        weitiandiDevice.get_collection().update({'mac':devDict['mac']},{'$set':devDict},upsert = True)
    except Exception,e:
        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!',e
        continue
print('OK')