trivial

script/spider/58.py

@@ -1,131 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-CITY_MAP = {
-    'hf': u'合肥'
-}
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '127.0.0.1',
-                    port = 27017)
-
-
-class Village(DynamicDocument):
-    city = StringField(default = "")
-    area = StringField(default = "")
-    name = StringField(default = '')
-    type = StringField(default = "")
-    address = StringField(default = "")
-    location = PointField(default = None)
-    alias = StringField(default = '')
-
-    meta = {
-        'collection': 'village',
-        'db_alias': 'spider',
-        'unique_together': {'city', 'area', 'name'}
-    }
-
-
-def get_location(name):
-    url_template = 'https://apis.map.qq.com/jsapi?qt=geoc&addr={}&key=FBOBZ-VODWU-C7SVF-B2BDI-UK3JE-YBFUS&output=jsonp&pf=jsapi&ref=jsapi'
-    url = url_template.format(urllib.quote(name))
-
-    print url
-
-    strhtml = requests.get(url, timeout = 15).text
-    result = json.loads(strhtml)
-
-    print result
-
-    longitude = float(result['detail']['pointx'])
-    latitude = float(result['detail']['pointy'])
-    return {
-        'type': 'Point', 'coordinates': [longitude, latitude]
-    }
-
-def get_location2(name):
-    url_template = 'https://apis.map.qq.com/jsapi?qt=poi&wd={}&pn=0&rn=10&rich_source=qipao&rich=web&nj=0&c=1&key=FBOBZ-VODWU-C7SVF-B2BDI-UK3JE-YBFUS&pf=jsapi&ref=jsapi'
-    url = url_template.format(name)
-
-    strhtml = requests.get(url, timeout = 15).text
-    result = json.loads(strhtml)
-
-    try:
-        poi = result['detail']['pois'][0]
-
-        longitude = float(poi['pointx'])
-        latitude = float(poi['pointy'])
-        return {
-            'type': 'Point', 'coordinates': [longitude, latitude]
-        }
-    except Exception as e:
-        print e.message
-        print result
-        print url.encode('utf-8')
-
-def spider_one_city(city, local_name):
-    curr_page = 1
-    total_page = 9999
-
-    while curr_page < total_page:
-        print 'curr = {}, total = {}'.format(curr_page, total_page)
-
-        url = 'https://m.58.com/xiaoquweb/getXiaoquList/?city={city}&key=&page={page}&price=&sort=&completiontime=&latlon=&stationid='.format(
-            city = city, page = curr_page)
-        strhtml = requests.get(url, timeout = 15).text
-
-        result = json.loads(strhtml)
-
-        dto_page = result['data']['pageDTO']
-
-        if int(dto_page['totalPage']) > total_page or total_page == 9999:
-            total_page = int(dto_page['totalPage'])
-
-        info_list = result['data']['infoList']
-        for item in info_list:
-            address = item['address']
-            alias = item['alias']
-            area = item['areaName']
-            village_type = item['infoParamEntity']['map']['propertytype']
-            name = item['name']
-
-            try:
-                village = Village(city = local_name, area = area, name = name, type = village_type, address = address,
-                                  alias = alias)
-                village.save()
-            except Exception as e:
-                print e.message
-
-        curr_page = curr_page + 1
-
-
-try:
-    spider_one_city('su', u'苏州')
-except Exception as e:
-    print e.message
-
-items = Village.objects.filter(city = u'苏州', location = None)
-for item in items:
-    try:
-        find_name = u'{} {} {}'.format(item.city, item.area, item.name)
-        item.location = get_location2(find_name.encode('utf8'))
-        item.save()
-    except Exception as e:
-        print e.message

script/spider/baiduStationExport.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class ChargeStation(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    address = StringField(default = '')
-    telephone = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '')
-    more = StringField(default = '')
-    
-    meta = {
-        'collection': 'ChargeStation',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-
-
-records = []
-for info in ChargeStation.objects.all():
-    dataList = [
-        (u'province', info.province),
-        (u'city', info.city),
-        (u'name', info.name),
-        (u'address', info.address),
-        (u'telephone', info.telephone),
-        (u'more', info.more),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/baiduGaodeStation.xlsx', records,True)
-print('OK')

script/spider/chaoxiang.py

@@ -1,297 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField,Document
-import simplejson as json
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-# from django.db.models.fields import DateTimeField
-# 
-# 
-# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-# 
-# from script.base import init_env
-# 
-# init_env(interactive = False)
-
-import pyautogui
-import qrcode
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-
-# 防止https证书校验不正确
-import ssl
-
-import pytesseract
-from PIL import Image
-
-# 超翔的需要利用android手机，打开小程序，然后拿到数据。我直接用的影刀连接android手机（必须调试模式，运行模拟点击的调试模式），然后再通过PC自动化进行操作
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class chaoxiang(Document):    
-    devNo = StringField(default = '')
-    phone = StringField(default = '')
-
-    meta = {
-        'collection': 'chaoxiang',
-        'db_alias': 'spider',
-        'unique_together': {'devNo'}
-    }
-    
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-# 拼接通用文字识别高精度url
-#
-
-def get_zone_text(region):
-    file1 = pyautogui.screenshot(region=region)
-    picPath = u'Q:/temp_chaoxiang/' + '%s.jpg' % int(time.time()*1000)
-    file1.save(picPath)
-    string = pytesseract.image_to_string(Image.open(picPath), lang='chi_sim')
-    return string.replace(' ','')
-    
-def back_to_search():
-    string = get_zone_text(region=(782,183,228,27))
-    if  u'启动设备' in string:
-        pyautogui.click(x=1265, y=436)
-        time.sleep(1)
-    elif u'搜索' in string:
-        return True
-    elif u'通话' in string:
-        pyautogui.click(x=1265, y=436)
-        time.sleep(1)
-        pyautogui.click(x=1265, y=436)
-        time.sleep(1)
-    elif u'首页' in string:
-        pyautogui.click(x=1090, y=241)
-        time.sleep(1)
-    
-    string = get_zone_text(region=(782,183,228,27))
-    if u'搜索' in string:
-        return True
-    else:
-        return False
-        
-def get_text_from_baidu(region):
-    try:
-        file1 = pyautogui.screenshot(region=region)
-        picPath = u'Q:/temp_chaoxiang/' + '%s.jpg' % int(time.time()*1000)
-        file1.save(picPath)
-        
-        f = open(picPath, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        request_url1 = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url1, data=params, headers=headers)
- 
-        # 解析返回结果
-        result_json = response.json()
-         
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            return text
-        return ''
-    except Exception,e:
-        print u"解析图片报错了！！！！",e
-#         
-# string = get_text_from_baidu(None)
-
-for ii in range(0,259889):
-    jj = 259889 - ii
-    if jj < 100000:
-        break
-    print jj
-
-    shouji = pyautogui.getWindowsWithTitle(u'手机管理器')[0]
-    if not shouji:
-        break
-    shouji.activate()
-    time.sleep(3)
-    
-    if not back_to_search():
-        print u'无法定位到搜索界面'
-        break
-    
-    # 点击搜索框
-    pyautogui.doubleClick(x=1066, y=249,interval=0.1)
-    time.sleep(1)
-     
-    # 点击删除按钮，把可能的多余的删除掉
-    pyautogui.click(x=1172, y=806,clicks=10,interval=0.1)
-     
-    time.sleep(0.1)
-     
-    # 输入设备ID号
-    pyautogui.write("%s" % jj)
-    time.sleep(0.1)
-     
-    # 点击搜索
-    pyautogui.click(x=1170, y=868)
-    time.sleep(1)
-     
-    # 检查是否有搜索结果
-    string = get_zone_text(region=(828,316,332,100))
-    print string
-    if u'无' in string:
-        print u'没有找到此设备',jj
-        continue
-    
-    # 点击设备，多等下，因为广告比较慢
-    pyautogui.click(x=1069, y=343)
-    time.sleep(5)
-    
-    # 点击可能打开的广告，没有广告，多点击一次也无所谓
-    pyautogui.click(x=1003,y=197)
-    time.sleep(0.5)
-     
-    # 检查是否有联系商户，没有直接返回
-    string = get_text_from_baidu(region=(1138,765,61,32))
-    if not u'联系' in string:
-        print u'没有找到联系方式按钮'
-        result = back_to_search()
-        if result:
-            continue
-        else:
-            print u'，请人工干预'
-            break
-    
-    # 点击联系方式
-    pyautogui.click(x=1167,y=775)
-    time.sleep(1)
-    
-    #解析电话
-    string = get_zone_text(region=(813,431,331,46))
-    try:
-        newObj = chaoxiang(str(jj),string)
-        newObj.save()
-        print u'搞到一条信息!!!',string
-    except Exception,e:
-        continue
-    
-    result = back_to_search()
-    if result:
-        continue
-    else:
-        print u'无法返回到搜索页面，请人工干预'
-        break
-#     time.sleep(0.1)
-# 
-# 
-# while True:
-#     a= auto.GetFocusedControl()
-#     time.sleep(2)
-#     print a
-#     print a.Name
-#     time.sleep(2)
-
-
-# while True:
-#     pyautogui.moveTo(987,574)
-#     file1 = pyautogui.screenshot(region=(766,220,412,575))
-#     file1.save(u'Q:/友商信息/汽车桩/星星充电/站列表截图/%s.png' % int(time.time()*1000))
-#     ii += 1
-#     print ii
-#     pyautogui.scroll(-400)
-#     time.sleep(0.2)
-    
-print 'OK'

script/spider/chaoxiang_get_deviceno_from_list.py

@@ -1,264 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-import pyautogui
-import qrcode
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-
-# 防止https证书校验不正确
-import ssl
-
-# while True:
-#     a = pyautogui.getWindowsWithTitle(u'图片查看')
-#     time.sleep(1)
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class yongxing(Searchable):    
-    devNo = StringField(default = '')
-    phone = StringField(default = '')
-
-    meta = {
-        'collection': 'yongxing',
-        'db_alias': 'spider',
-        'unique_together': {'devNo'}
-    }
-    
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-# 拼接通用文字识别高精度url
-#
-def get_phone(devNo,path):
-    try:
-        f = open(path, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        request_url1 = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url1, data=params, headers=headers)
-
-        # 解析返回结果
-        result_json = response.json()
-        
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            
-            if u'打' in text and len(text)>7:
-                phone = text.replace(u'打',"")
-                print u'获取到一条电话！！',devNo,phone
-                try:
-                    yongxing(devNo,phone).save()
-                    break
-                except Exception,e:
-                    break
-        
-    except Exception,e:
-        print u"解析图片报错了！！！！",e
-        
-    
-
-moniqi = pyautogui.getWindowsWithTitle(u'逍遥模拟器')[0]
-
-if not moniqi:
-    break
-moniqi.activate()
-
-# 点击文件按钮
-pyautogui.click(x=762, y=767)
-time.sleep(0.2)
-   
-# 输入二维码文件名称
-pyautogui.click(x=1083, y=928)
-pyautogui.write("%s" % jj)
-time.sleep(0.2)
-   
-# 选择该文件,传到编辑框
-with pyautogui.hold('alt'):
-    pyautogui.press('o')
-time.sleep(0.1)
-
-# 双击打开二维码
-pyautogui.doubleClick(x=791, y=816)
-time.sleep(1)
-
-# 右键打开
-pyautogui.rightClick(x=983,y=468)
-time.sleep(0.1)
-# 点击识别二维码
-pyautogui.click(x=1084,y=553)
- 
-# 等待小程序启动界面出来.如果
-time.sleep(1.7)
-title = pyautogui.getActiveWindowTitle()
-timeCount = 0
-if title != u'富连e充':
-    print u" 无法拉起来小程序"
-    time.sleep(1800)
-
-# 截图
-file1 = pyautogui.screenshot(region=(758,113,402,765))
-picPath = u'Q:/友商信息/涌鑫/小程序截图/' + '%s.png' % int(time.time()*1000)
-file1.save(picPath)
-get_phone('1000%04d' % jj, picPath)
-
-#  加按几次esc，用于消掉一些提示
-pyautogui.press('esc')
-time.sleep(0.5)
-
-#     # 点击关闭小程序
-#     pyautogui.click(x=1135, y=124)
-#     time.sleep(0.1)
-#     
-#     # 关闭截图
-#     pyautogui.click(x=983,y=468)
-#     time.sleep(0.2)
-#     pyautogui.press('esc')
-#     time.sleep(0.2)
-
-# 删除截图
-#     pyautogui.click(x=791, y=816)
-weixin = pyautogui.getWindowsWithTitle(u'微信')[0]
-weixin.activate()
-
-pyautogui.click(x=1413, y=800)
-time.sleep(0.5)
-
-with pyautogui.hold('ctrl'):
-    pyautogui.press('a')
-pyautogui.press('delete')
-#     time.sleep(0.1)
-# 
-# 
-# while True:
-#     a= auto.GetFocusedControl()
-#     time.sleep(2)
-#     print a
-#     print a.Name
-#     time.sleep(2)
-
-
-# while True:
-#     pyautogui.moveTo(987,574)
-#     file1 = pyautogui.screenshot(region=(766,220,412,575))
-#     file1.save(u'Q:/友商信息/汽车桩/星星充电/站列表截图/%s.png' % int(time.time()*1000))
-#     ii += 1
-#     print ii
-#     pyautogui.scroll(-400)
-#     time.sleep(0.2)
-    
-print 'OK'

script/spider/dongjun_tel.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-
-ownerDict = {}
-def from_exl(fileName,ownerDict):
-    book = xlrd.open_workbook(fileName)
-    
-    try:
-        sheet = book.sheet_by_name("dongjun_devices")
-    except Exception,e:
-        print 'open excel file error =%s' % e
-        exit(0)
-    
-    nrows = sheet.nrows
-    rows = []
-    for i in range(1, nrows):
-        row = sheet.row_values(i)
-        try:
-            mobile = str(row[13]).replace('\t','').upper()
-            if mobile in ownerDict:
-                ownerDict[mobile] += 1
-            else:
-                ownerDict[mobile] = 1
-        except Exception,e:
-            print 'some error when update iccid=%s,e=%s' % (row[1],e)
-
-from_exl(u'Q:\友商信息\友商分析\dongjun_devices2.xls', ownerDict)
-from_exl(u'Q:\友商信息\友商分析\dongjun_devices3.xls', ownerDict)
-from_exl(u'Q:\友商信息\友商分析\dongjun_devices4.xls', ownerDict)
-
-records = []
-for mobile,count in ownerDict.items():
-    dataList = [
-        (u'联系', mobile),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/dongjun_tel_count.xlsx', records,True)
-
-print('OK')

script/spider/haokuaichong.py

@@ -1,96 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class haokuaichongDevice(Searchable):
-    meta = {
-        'collection': 'haokuaichong_device',
-        'db_alias': 'spider',
-    }
-
-class haokuaichongOtherDevice(Searchable):
-    meta = {
-        'collection': 'haokuaichong_other_device',
-        'db_alias': 'spider',
-    }
-    
-# 分析公众号
-# url = 'http://ap3202.haokuaichong.cn/quickuser50/charge/info?device=%s&port=FF&cmd=0'
-# cookie = 'JSESSIONID=8E5A27C36409661680FCADD921EF267A;tgw_l7_route=8601cd2f2bac0d2b07a50c969c168416'
-# auth = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoidXNlciIsImVuY3J5SWQiOiJOakEzTXpNMU1RPT0iLCJpc3MiOiJxdWlja191c2VyXzAwMSIsImlhdCI6MTY0OTQwMDIyNiwiYXVkIjoicXVpY2tfdXNlcl9hcGkiLCJleHAiOjE2NDk0MDM4MjYsIm5iZiI6MTY0OTQwMDIyNn0.C1KXB77mjkhsof01NGFoWM_TdSLeZ3OlWVuIiqLZXAw'
-# for ii in range(80,290000): #00,01,02,03
-#     logicalId = ('%02xE1A704' % ii).upper()
-#     url1 = url % logicalId 
-#     strhtml = requests.get(url1,headers = {'Cookie':cookie,'Authorization':auth}).text
-#     result = json.loads(strhtml)
-#     print result
-#     if result['msg'] == u'请勿跨公众号使用':
-#         print '------------------->',logicalId
-        
-        
-# 根据二维码编号，获取设备更详细的信息
-url = 'http://ap3202.haokuaichong.cn/quickuser50/charge/info?device=%s&port=00&cmd=1'
-cookie = 'JSESSIONID=8E5A27C36409661680FCADD921EF267A;tgw_l7_route=8601cd2f2bac0d2b07a50c969c168416'
-auth = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoidXNlciIsImVuY3J5SWQiOiJOakUzTVRVMU5nPT0iLCJpc3MiOiJxdWlja191c2VyXzAwMSIsImlhdCI6MTY0OTkwMzI0OSwiYXVkIjoicXVpY2tfdXNlcl9hcGkiLCJleHAiOjE2NDk5MDY4NDksIm5iZiI6MTY0OTkwMzI0OX0.CN-UApjFt7mPChSIxHspf7gQoIM0rd6wZmefVe3_FHo'
-for ii in range(0,390000): #7174
-    try:
-        logicalId = '11%06d' % ii
-        url1 = url % logicalId 
-        strhtml = requests.get(url1,headers = {'Cookie':cookie,'Authorization':auth}).text
-        result = json.loads(strhtml)
-        if result['code'] == 500 :
-            if result['msg'] == u'请勿跨公众号使用':
-                haokuaichongOtherDevice.get_collection().update({'deviceId':logicalId},{'$set':{'deviceId':logicalId}},upsert = True)
-                continue
-            else:
-                continue
-        if result['code'] == 70004:
-            print ii, 'token 过期'
-            break
-        dev = result['data']
-        
-        haokuaichongDevice.get_collection().update({'deviceId':dev['deviceId']},{'$set':dev},upsert = True)
-        print ii,'catch new one!!!!'
-    except Exception,e:
-        print 'ExceptionExceptionExceptionExceptionException',e
-        continue
-print('OK')

script/spider/haokuaichong2.py

@@ -1,99 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from bson import ObjectId
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class haokuaichongDevice(Searchable):
-    meta = {
-        'collection': 'haokuaichong_device',
-        'db_alias': 'spider',
-    }
-
-class haokuaichongOtherDevice(Searchable):
-    meta = {
-        'collection': 'haokuaichong_other_device',
-        'db_alias': 'spider',
-    }
-    
-# 分析公众号
-# url = 'http://ap3202.haokuaichong.cn/quickuser50/charge/info?device=%s&port=FF&cmd=0'
-# cookie = 'JSESSIONID=8E5A27C36409661680FCADD921EF267A;tgw_l7_route=8601cd2f2bac0d2b07a50c969c168416'
-# auth = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoidXNlciIsImVuY3J5SWQiOiJOakEzTXpNMU1RPT0iLCJpc3MiOiJxdWlja191c2VyXzAwMSIsImlhdCI6MTY0OTQwMDIyNiwiYXVkIjoicXVpY2tfdXNlcl9hcGkiLCJleHAiOjE2NDk0MDM4MjYsIm5iZiI6MTY0OTQwMDIyNn0.C1KXB77mjkhsof01NGFoWM_TdSLeZ3OlWVuIiqLZXAw'
-# for ii in range(80,290000): #00,01,02,03
-#     logicalId = ('%02xE1A704' % ii).upper()
-#     url1 = url % logicalId 
-#     strhtml = requests.get(url1,headers = {'Cookie':cookie,'Authorization':auth}).text
-#     result = json.loads(strhtml)
-#     print result
-#     if result['msg'] == u'请勿跨公众号使用':
-#         print '------------------->',logicalId
-
-
-    
-# 根据二维码编号，获取设备更详细的信息
-url = 'http://ap3202.haokuaichong.cn/quickuser50/charge/info?device=%s&port=00&cmd=1'
-cookie = 'JSESSIONID=8E5A27C36409661680FCADD921EF267A;tgw_l7_route=8601cd2f2bac0d2b07a50c969c168416'
-auth = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJyb2xlIjoidXNlciIsImVuY3J5SWQiOiJOak0wT0RVME13PT0iLCJpc3MiOiJxdWlja191c2VyXzAwMSIsImlhdCI6MTY1MTE0MDc0MywiYXVkIjoicXVpY2tfdXNlcl9hcGkiLCJleHAiOjE2NTExNDQzNDMsIm5iZiI6MTY1MTE0MDc0M30.5C4E1fqe-C73SFVIORYBKhUb5pxmNrl-p0ukuT2_Ll8'
-devIds = [rcd for rcd in haokuaichongOtherDevice.get_collection().find({'projectId':None})]#,'_id':{'$gt':ObjectId("6253f82e2be18bf0dc979429")}})]
-ii = 0
-for devIdObj in devIds[5000:]: #7174
-    devId = devIdObj['deviceId']
-    ii += 1
-    if ii % 100 == 0:
-        print ii,devIdObj['_id']
-    try:
-        url1 = url % devId 
-        strhtml = requests.get(url1,headers = {'Cookie':cookie,'Authorization':auth}).text
-        result = json.loads(strhtml)
-        if result['code'] == 500 :
-            continue
-        if result['code'] == 70004:
-            print ii, 'token 过期',devIdObj['_id']
-            break
-        dev = result['data']
-        dev['code'] = devId
-        haokuaichongOtherDevice.get_collection().update({'deviceId':devId},dev,upsert = True)
-        print ii,'catch new one!!!!'
-    except Exception,e:
-        print 'ExceptionExceptionExceptionExceptionException',e
-        continue
-print('OK')

script/spider/haokuaichongTel.py

@@ -1,77 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class haokuaichongDevice(Searchable):
-    meta = {
-        'collection': 'haokuaichong_device',
-        'db_alias': 'spider',
-    }
-
-class haokuaichongOtherDevice(Searchable):
-    meta = {
-        'collection': 'haokuaichong_other_device',
-        'db_alias': 'spider',
-    }
-    
-ownerDict = {}
-for dev in haokuaichongDevice.get_collection().find():
-    if dev['phone'] in ownerDict:
-        ownerDict[dev['phone']] += 1
-    else:
-        ownerDict[dev['phone']] = 1
-        
-records = []
-for mobile,count in ownerDict.items():
-    dataList = [
-        (u'联系', mobile),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/haokuaichong1.xlsx', records,True)
-print('OK')

script/spider/mashangchong.py

@@ -1,63 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class shankailaidianDevice(Searchable):
-    meta = {
-        'collection': 'shankailaidian_device',
-        'db_alias': 'spider',
-        'unique_together': {'vStationNo'}
-    }
-
-# 根据二维码编号，获取设备更详细的信息
-url1 = 'https://m.mashangchong.com.cn/rest/dev/'
-
-
-cookie1 = 'csrftoken=68md9lvb8A1V4B0oms6QwXQMY8mOuM7F9jqAPj1FXy9HEbcZahZPN3vqp549A1eo; sessionid=dhb9a7k0zi3lzjfmzmin3rgto6hkegek'
-agent = 'Mozilla/5.0 (Linux; Android 7.1.2; HLTE212T Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/86.0.4240.99 XWEB/3170 MMWEBSDK/201101 Mobile Safari/537.36 MMWEBID/6505 MicroMessenger/7.0.21.1800(0x2700157F) Process/toolsmp WeChat/arm32 Weixin NetType/WIFI Language/zh_CN ABI/arm64'
-refer = 'https://api.issks.com/issksapi/V2/ec/stationList.shtml'
-token = '68md9lvb8A1V4B0oms6QwXQMY8mOuM7F9jqAPj1FXy9HEbcZahZPN3vqp549A1eo'
-url = url1
-strhtml = requests.get(url,headers = {'Cookie':cookie1,'User-Agent':agent,'X-CSRFToken':token}).text
-result = json.loads(strhtml)
-        
-print('OK')

script/spider/pianyichong.py

@@ -1,83 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class pianyichongDevice(Searchable):
-    meta = {
-        'collection': 'pianyichong_device',
-        'db_alias': 'spider',
-        'unique_together': {'id'}
-    }
-
-# 根据二维码编号，获取设备更详细的信息
-url1 = 'http://www.bianyichong.cn/prod/prodInstlPos/queryGeo'
-ii = 43570
-govs = Goverment.get_collection().find()
-
-devIdMap = {}
-for dev in pianyichongDevice.get_collection().find({}):
-    devIdMap[dev['id']] = 1
-
-for gov in govs[ii:]:
-    try:
-        ii += 1
-        print ii
-        if not (u'政府' in gov['name'] or u'局' in gov['name']):
-            continue
-        
-        data = {'lng':gov['lng'],'lat':gov['lat'],'bizNo':'B100000','distance':3000}
-        strhtml = requests.post(url1,data=data).text
-        result = json.loads(strhtml)
-        if not result.has_key('model'):
-            continue
-        if not result['model'].has_key('prodInstlPosList') or len(result['model']['prodInstlPosList']) == 0:
-            continue
-        for dev in result['model']['prodInstlPosList']:
-            if dev['id'] in devIdMap:
-                continue
-            pianyichongDevice.get_collection().update({'id':dev['id']},{'$set':dev},upsert = True)
-            devIdMap[dev['id']] = 1
-            print 'new one!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
-    except Exception,e:
-        print 'eeeeeeeeeeeeeeeeeeeeeee',e
-        continue
-print('OK')

script/spider/pianyichongExport.py

@@ -1,105 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class pianyichongDevice(Searchable):
-    meta = {
-        'collection': 'pianyichong_device',
-        'db_alias': 'spider',
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-ownerDict = {}
-for shopInfo in pianyichongDevice.get_collection().find():
-    if not shopInfo.has_key('manager'):
-        continue
-    if ownerDict.has_key(shopInfo['manager']):
-        ownerDict[shopInfo['manager']] += 1
-    else:
-        ownerDict[shopInfo['manager']] = 1
-
-records = []
-for shop,count in ownerDict.items():
-#     zone = get_tel_zone(mobile)
-#     print zone[0],zone[1],mobile,count
-#     city,contact = '',''
-#     tempList = shop.split(' ')
-#     if len(tempList) > 2:
-#         city,contact = tempList[0],tempList[1]
-    dataList = [
-        (u'通讯', shop),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/pianyichong.xlsx', records,True)
-print('OK')

script/spider/print_mouse_position.py

@@ -1,9 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import time
-import pyautogui
-
-while True:
-    print pyautogui.position()
-    time.sleep(1)

script/spider/shankailaidian.py

@@ -1,86 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class shankailaidianDevice(Searchable):
-    meta = {
-        'collection': 'shankailaidian_device',
-        'db_alias': 'spider',
-        'unique_together': {'vStationNo'}
-    }
-
-# 根据二维码编号，获取设备更详细的信息
-url1 = 'https://api.issks.com/issksapi/V2/ec/stationList/json.shtml?mapX=%s&mapY=%s'
-ii = 0
-govs = Goverment.get_collection().find({})
-
-devIdMap = {}
-for dev in shankailaidianDevice.get_collection().find({}):
-    devIdMap[dev['vStationNo']] = 1
-
-cookie1 = 'SECKEY_ABVK=9Evaq2OKVzgoe1gStw9X4D1UBWkr2l7/0NRT+NUeCJM%3D; BMAP_SECKEY=zkVWrAyscY50tP8ct7mfV8gd7WUVrYi84X5_k1PtFRpswZA8LXI2sUKNQ0c8MUI2xqDaQuWJa5VqB3OiOZNn0zokd5VNvfOykp0gHv46O1UvEecnTCvXqGcz8Qu6CZMA6NcAEK-wGEIW8wIV2H1c4-Iq5avaGxXajjaUVgVgm1NvCcAaN9PAaz4276H_pE7d; JSESSIONID=DDA5CAD77862B1B692D5055FD8F4818B; 14a03328ed59437f0a5a7fea4391b872=3b935069ae8453531d5ebb3e151b6cac'
-agent = 'Mozilla/5.0 (Linux; Android 7.1.2; HLTE212T Build/N2G47H; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/86.0.4240.99 XWEB/3170 MMWEBSDK/201101 Mobile Safari/537.36 MMWEBID/6505 MicroMessenger/7.0.21.1800(0x2700157F) Process/toolsmp WeChat/arm32 Weixin NetType/WIFI Language/zh_CN ABI/arm64'
-refer = 'https://api.issks.com/issksapi/V2/ec/stationList.shtml'
-
-for gov in govs[ii:]:
-    try:
-        ii += 1
-        print gov['province'],gov['city'],gov['name'],ii
-        if not (u'政府' in gov['name']):
-            continue
-        
-        url = url1 % (gov['lng'],gov['lat'])
-        strhtml = requests.get(url,headers = {'Cookie':cookie1,'User-Agent':agent,'Refer':refer}).text
-        result = json.loads(strhtml)
-        if not result.has_key('list'):
-            continue
-        
-        for dev in result['list']:
-            if dev['vStationNo'] in devIdMap:
-                continue
-            shankailaidianDevice.get_collection().update({'vStationNo':dev['vStationNo']},{'$set':dev},upsert = True)
-            devIdMap[dev['vStationNo']] = 1
-            print 'new one!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
-    except Exception,e:
-        print 'eeeeeeeeeeeeeeeeeeeeeee',e
-        continue
-print('OK')

script/spider/shankailaidianAnalyze.py

@@ -1,84 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class shankailaidianDevice(Searchable):
-    meta = {
-        'collection': 'shankailaidian_device',
-        'db_alias': 'spider',
-        'unique_together': {'vStationNo'}
-    }
-
-
-
-records = []
-owerDict = {}
-ii = 0
-for rcd in shankailaidianDevice.get_collection().find({}):
-    ii += 1
-    if ii % 100 == 0:
-        print ii
-    
-    addYear = rcd['tAddTime'].split('-')[0]
-    
-    if rcd['vHelpMobile'] in owerDict:
-        owerDict[rcd['vHelpMobile']]['count'] += 1
-        owerDict[rcd['vHelpMobile']]['income'] += float(rcd['dTotalIncome'])
-        if owerDict[rcd['vHelpMobile']].has_key(addYear):
-            owerDict[rcd['vHelpMobile']][addYear] += 1
-        else:
-            owerDict[rcd['vHelpMobile']][addYear] = 1
-        
-    else:
-        owerDict[rcd['vHelpMobile']] = {'count':1,'income':float(rcd['dTotalIncome']),addYear:1}
-
-
-for mobile,info in owerDict.items():
-    ii += 1
-    if ii % 100 == 0:
-        print ii
-    dataList = [
-        (u'电话', mobile),
-        (u'数目', info['count']),
-        (u'总收入', info['income']),
-        (u'分配', info['income']*0.2),
-        (u'单台设备均分', info['income']*0.2/info['count']),
-        (u'2022', info.get('2022',0)),
-        (u'2021', info.get('2021',0)),
-        (u'2020', info.get('2020',0)),
-        (u'2019', info.get('2019',0)),
-        (u'2018', info.get('2018',0)),
-    ]
-    
-
-    records.append(OrderedDict(dataList))
-generate_excel_report('F:/shankailaidian_tongji3.xlsx', records,True)
-print('OK')

script/spider/shankailaidianExport.py

@@ -1,143 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class shankailaidianDevice(Searchable):
-    meta = {
-        'collection': 'shankailaidian_device',
-        'db_alias': 'spider',
-        'unique_together': {'vStationNo'}
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-# ownerDict = {}
-# for shopInfo in pianyichongDevice.get_collection().find():
-#     if not shopInfo.has_key('manager'):
-#         continue
-#     if ownerDict.has_key(shopInfo['manager']):
-#         ownerDict[shopInfo['manager']] += 1
-#     else:
-#         ownerDict[shopInfo['manager']] = 1
-
-# records = []
-# ii = 0
-# for rcd in shankailaidianDevice.get_collection().find({}):
-#     ii += 1
-#     if ii % 100 == 0:
-#         print ii
-#     dataList = [
-#         (u'编号', str(rcd['vStationNo'])),
-#         (u'总收入', float(rcd['dTotalIncome'])),
-#         (u'地域ID', str(rcd['iAreaId'])),
-#         (u'businessType', str(rcd['iBusinessType'])),
-#         (u'iManagerId', str(rcd['iManagerId'])),
-#         (u'添加时间', str(rcd['tAddTime'])),
-#         (u'打开时间', str(rcd['tOpenTime'])),
-#         (u'地址', str(rcd['vAddress'])),
-#         (u'地域', str(rcd['vAreaName'])),
-#         (u'电话', rcd['vHelpMobile'] if rcd['vHelpMobile'] != '' else rcd['vTelephone']),
-# #         (u'充电站名称', str(rcd['vStationName'])),
-#     ]
-# 
-#     records.append(OrderedDict(dataList))
-# 
-# generate_excel_report('F:/shankailaidian1.xlsx', records,True)
-
-
-records = []
-owerDict = {}
-ii = 0
-for rcd in shankailaidianDevice.get_collection().find({}):
-    ii += 1
-    if ii % 100 == 0:
-        print ii
-
-    if rcd['vHelpMobile'] in owerDict:
-        owerDict[rcd['vHelpMobile']]['count'] += 1
-        owerDict[rcd['vHelpMobile']]['income'] += float(rcd['dTotalIncome'])
-    else:
-        owerDict[rcd['vHelpMobile']] = {'count':1,'income':float(rcd['dTotalIncome'])}
-
-for mobile,info in owerDict.items():
-    ii += 1
-    if ii % 100 == 0:
-        print ii
-    dataList = [
-        (u'电话', mobile),
-        (u'数目', info['count']),
-        (u'总收入', info['income']),
-        (u'分配', info['income']*0.2),
-        (u'单台设备均分', info['income']*0.2/info['count']),
-    ]
-
-    records.append(OrderedDict(dataList))
-generate_excel_report('F:/shankailaidian_tongji2.xlsx', records,True)
-print('OK')

script/spider/test.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-def get_shops_info(lat,lng):
-    url = 'https://u.zhinengxiyifang.cn/api/Stores/findNearStoreInc'
-    payload = {"lat":lat,"lont":lng,"skip":0,"limit":10}
-#     url = 'https://www.baidu.com'
-#     payload = {}
-    proxies = {'http':'http://140.246.89.239:19382','https':'https://140.246.89.239:19382'}
-    strhtml = requests.post(url, json = payload,proxies = proxies,timeout = 15).text
-    result = json.loads(strhtml)
-    return result
-# cookies = {
-#            'JSESSIONID':'0626C177663F8C8D0D18D212DB4EF04F',
-#            'AppName':'%u6C5F%u897F%u7F8E%u5929%u65B0%u80FD%u6E90%u79D1%u6280%u6709%u9650%u516C%u53F8'
-#            }
-# url = 'https://1.yugu.net.cn/api/v1/agent/query_detail.htm'  # 'https://1.yugu.net.cn/api/v1/agent/detail.htm'
-# 
-# data = {'id':"327"}
-# strhtml = requests.post(url,json = data,headers = {'Authorization':'Bearer 2e8d745bae1e4a4561aacb2a4a733d9aade8'},verify=False,timeout = 15).text
-# result = json.loads(strhtml)
-
-# url  = 'http://cdz.gpsserver.cn/ChargeCarSys?gtel=68000000423'
-# strhtml = requests.get(url,timeout = 15).text
-# result = json.loads(strhtml)
-
-# url = 'http://ydc.youdianchong.com:8087/newCloseCharger?longitude=112.85811&latitude=35.496819'
-
-# url = 'https://1.yugu.net.cn/api/v1/agent/query_detail.htm'  # 'https://1.yugu.net.cn/api/v1/agent/detail.htm'
-# 
-# data = {'id':"327"}
-# strhtml = requests.get(url,headers = {'openId':'ozk7Zv4WC5akrk9Lvrm78jggwHk4'},verify=False,timeout = 15).text
-# result = json.loads(strhtml)
-
-import base64
-import sys
-import json
-import base64
-
-
-
-
-
-
-
-# 打印文字
-print('aaa')

script/spider/tiantian_pic_2_port.py

@@ -1,174 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-from django.db.models.fields import DateTimeField
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-
-# 防止https证书校验不正确
-import ssl
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class tiantianPort(Searchable):    
-    addr = StringField(default = '')
-    portId = StringField(default = '')
-    
-    meta = {
-        'collection': 'tiantian_port',
-        'db_alias': 'spider',
-        'unique_together': {'portId'}
-    }
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-# 拼接通用文字识别高精度url
-
-rootdir = u'F:/爬虫相关/天天充电/端口/'
-listFile = os.listdir(rootdir)  # 列出文件夹下所有的目录与文件
-for i in range(0, len(listFile)):
-    print i
-    try:
-        path = os.path.join(rootdir, listFile[i])
-        f = open(path, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        access_token = token
-        request_url = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url, data=params, headers=headers)
-#         if response:
-#             print (response.json())
-    
-        # 解析返回结果
-        result_json = response.json()
-        addr = ''
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            zeroIndex = text.find('0')
-            if zeroIndex >= 0 :
-                addr += text[0:zeroIndex]
-                portId = text[zeroIndex::]
-                tiantianPort.get_collection().update({'portId':portId},{'$set':{'portId':portId,'addr':addr}},upsert = True)
-                addr = ''
-            else:
-                addr = text
-    except Exception:
-        continue
-# 打印文字
-print('OK')

script/spider/tiantian_zone.py

@@ -1,71 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-from django.db.models.fields import DateTimeField
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-import qrcode
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class tiantianPort(Searchable):    
-    addr = StringField(default = '')
-    portId = StringField(default = '')
-    
-    meta = {
-        'collection': 'tiantian_port',
-        'db_alias': 'spider',
-        'unique_together': {'portId'}
-    }
-
-ports = [port for port in tiantianPort.objects.all()]
-zoneDict = {}
-for port in ports:
-    if not zoneDict.has_key(port.addr):
-        zoneDict[port.addr] = port
-
-
-for zone,port in zoneDict.items():
-    try:
-        qrCode = 'https://cd1a.cn/?s='+port.portId
-        print zone,port.portId,qrCode
-        img = qrcode.make(qrCode)
-        img.save('F:/tiantian/%s.jpg' % port.portId)
-    except Exception,e:
-        continue
-# 打印文字
-print('OK')

script/spider/ujing.py

@@ -1,93 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-def get_shops_info(lat,lng):
-    url = 'https://u.zhinengxiyifang.cn/api/Stores/findNearStoreInc'
-    payload = {"lat":lat,"lont":lng,"skip":0,"limit":10}
-#     url = 'https://www.baidu.com'
-#     payload = {}
-    proxies = {'http':'http://140.246.89.239:19382','https':'https://140.246.89.239:19382'}
-    strhtml = requests.post(url, json = payload,proxies = proxies,timeout = 15).text
-    result = json.loads(strhtml)
-    return result
-
-class School(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'School',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-
-class ujing(Searchable):
-    meta = {
-        'collection': 'ujing',
-        'db_alias': 'spider',
-        'unique_together': {'_id'}
-    }
-
-class spiderLog(Searchable):
-    lat = StringField(default = '')
-    lng = StringField(default = '')
-    addedTime = DateTimeField(default = datetime.datetime.now())
-    meta = {
-        'collection': 'spiderLog',
-        'db_alias': 'spider',
-        'unique_together': {'lat','lng'}
-    }
-    
-def upsert_shops(shopsInfo):
-    if not (shopsInfo.has_key('data') and shopsInfo['data'].has_key('storeList') and len(shopsInfo['data']['storeList']) > 0):
-        return
-    for shop in shopsInfo['data']['storeList']:
-        try:
-            ujing.get_collection().update({'_id':shop['_id']},{'$set':shop},upsert = True)
-        except Exception,e:
-            continue
-        
-locations = School.get_collection().find()
-for loc in locations:
-    try:
-        count = spiderLog.objects.filter(lat = loc['lat'],lng = loc['lng']).count()
-        if count > 0:
-            continue
-        shopsInfo = get_shops_info(loc['lat'], loc['lng'])
-        spiderLog(lat = loc['lat'],lng = loc['lng']).save()
-        print shopsInfo
-        time.sleep(1)
-        upsert_shops(shopsInfo)
-    except Exception,e:
-        print('error!!!!!!!!!!!!!!!!!!!',e)
-        continue

script/spider/ujing_stat.py

@@ -1,94 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-# 统计U净用户的数据
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class ujing(Searchable):
-    meta = {
-        'collection': 'ujing',
-        'db_alias': 'spider',
-        'unique_together': {'_id'}
-    }
-
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-ownerDict = {}
-for shopInfo in ujing.get_collection().find():
-    if ownerDict.has_key(shopInfo['mobile']):
-        ownerDict[shopInfo['mobile']] += shopInfo.get('deviceCount',0)
-    else:
-        ownerDict[shopInfo['mobile']] = shopInfo.get('deviceCount',0)
-
-records = []
-for mobile,count in ownerDict.items():
-    zone = get_tel_zone(mobile)
-    print zone[0],zone[1],mobile,count
-    dataList = [
-        (u'省份', zone[0]),
-        (u'城市', zone[1]),
-        (u'通讯', mobile),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test1.xlsx', records,False)

script/spider/wanzhuangExport.py

@@ -1,110 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class wanzhuangDevice(Searchable):
-    meta = {
-        'collection': 'wanzhuang_device',
-        'db_alias': 'spider',
-        'unique_together': {'device_num'}
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-ownerDict = {}
-for shopInfo in wanzhuangDevice.get_collection().find():
-    if not shopInfo.has_key('manager_phone'):
-        continue
-    if ownerDict.has_key(shopInfo['manager_phone']):
-        ownerDict[shopInfo['manager_phone']] += 1
-    else:
-        ownerDict[shopInfo['manager_phone']] = 1
-
-records = []
-for mobile,count in ownerDict.items():
-    zone = get_tel_zone(mobile)
-    print zone[0],zone[1],mobile,count
-#     city,contact = '',''
-#     tempList = shop.split(' ')
-#     if len(tempList) > 2:
-#         city,contact = tempList[0],tempList[1]
-    dataList = [
-        (u'省份', zone[0]),
-        (u'市', zone[1]),
-        (u'通讯', mobile),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test8.xlsx', records,False)
-print('OK')

script/spider/wanzhuangXiaochengxun.py

@@ -1,285 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField,Document
-import simplejson as json
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-# from django.db.models.fields import DateTimeField
-# 
-# 
-# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-# 
-# from script.base import init_env
-# 
-# init_env(interactive = False)
-
-import pyautogui
-import qrcode
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-
-# 防止https证书校验不正确
-import ssl
-import pytesseract
-from PIL import Image
-
-# while True:
-#     a = pyautogui.getWindowsWithTitle(u'图片查看')
-#     time.sleep(1)
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class wangzhuang(Document):    
-    devNo = StringField(default = '')
-    phone = StringField(default = '')
-
-    meta = {
-        'collection': 'wangzhuangXiaochengxu',
-        'db_alias': 'spider',
-        'unique_together': {'devNo'}
-    }
-    
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-
-def get_zone_text(region):
-    file1 = pyautogui.screenshot(region=region)
-    picPath = u'Q:/temp_chaoxiang/' + '%s.jpg' % int(time.time()*1000)
-    file1.save(picPath)
-    string = pytesseract.image_to_string(Image.open(picPath), lang='chi_sim')
-    print picPath,string
-    return string.replace(' ','')
-
-def get_text_from_baidu(region):
-    try:
-        file1 = pyautogui.screenshot(region=region)
-        picPath = u'Q:/temp_chaoxiang/' + '%s.jpg' % int(time.time()*1000)
-        file1.save(picPath)
-        
-        f = open(picPath, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        request_url1 = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url1, data=params, headers=headers)
- 
-        # 解析返回结果
-        result_json = response.json()
-         
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            return text
-        return ''
-    except Exception,e:
-        print u"解析图片报错了！！！！",e
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-        
-def back_to_message():
-    stringTitle = get_zone_text(region=(886,180,170,35))
-    stringLeft = get_zone_text(region=(782,183,228,27))
-    if  u'陈军' in stringTitle:
-        return True
-    
-    elif u'充电支付' in stringLeft:
-        pyautogui.click(x=1167, y=198) # 直接点击关闭按钮
-        time.sleep(1)
-        
-    elif u'通话' in stringLeft:
-        pyautogui.click(x=1265, y=436)
-        time.sleep(1)
-        pyautogui.click(x=1265, y=436)
-        time.sleep(1)
-    
-    stringTitle = get_zone_text(region=(886,180,170,35))
-    if u'陈军' in stringTitle:
-        return True
-    else:
-        return False
-
-# 从30万启动开始
-for ii in range(0,300000):
-    jj = 300000 - ii
-    print jj
-    qrStr = 'https://www.ctiot.info/d/18%06d' % jj
-    img =qrcode.make(qrStr)
-    filePath = "Q:/temp/%s" % jj
-    img.save(filePath)
-    time.sleep(0.2)
-        
-    weixin = pyautogui.getWindowsWithTitle(u'微信')[0]
-    if not weixin:
-        break
-    weixin.activate()
-    time.sleep(3)
-     
-    # 点击文件按钮
-    pyautogui.click(x=761, y=777)
-    time.sleep(0.2)
-        
-    # 输入二维码文件名称
-    pyautogui.click(x=1083, y=928)
-    pyautogui.write("%s" % jj)
-    time.sleep(0.2)
-        
-    # 选择该文件,传到编辑框
-    with pyautogui.hold('alt'):
-        pyautogui.press('o')
-    time.sleep(0.1)
-     
-    # 发消息
-    with pyautogui.hold('alt'):
-        pyautogui.press('s')
-    time.sleep(0.1)
-    
-    # --------------------------------- 手机这边识别二维码-------------------------
-    shouji = pyautogui.getWindowsWithTitle(u'手机管理器')[0]
-    if not shouji:
-        break
-    shouji.activate()
-    time.sleep(3)
-    
-    if not back_to_message():
-        print u'无法定位到消息接受窗口'
-        break
-    
-    # 点击图片
-    pyautogui.click(x=1067, y=782)
-    time.sleep(1)
-    
-    # 点击图片的二维码识别按钮
-    pyautogui.click(x=1025, y=864)
-    time.sleep(3)
-    
-    string = get_zone_text(region=(782,183,228,27))
-    if u'充电支付' not in string:
-        print u'没有启动小程序，需要等等吧'
-        time.sleep(1800)
-        
-    string = get_zone_text(region=(847,568,82,30))
-    if u'消' in string:
-        pyautogui.click(x=847, y=568)
-        time.sleep(0.2)
-    
-#     devNo = get_text_from_baidu(region=(855,278,68,21))
-#     if devNo and devNo != '18%06d' % jj:
-#         print u'设备编号错误，可能是二维码没有发过来',devNo,jj
-#         break
-    
-    string = get_text_from_baidu(region=(1122,262,72,40))
-    if not u'电话' in string :
-        continue 
-    
-    # 点击联系方式
-    pyautogui.click(x=1154, y=278)
-    time.sleep(1)
-    
-    #解析电话
-    string = get_zone_text(region=(813,431,331,46))
-    if not string:
-        continue
-    try:
-        newObj = wangzhuang(str(jj),string)
-        newObj.save()
-        print u'搞到一条信息!!!',string
-    except Exception,e:
-        continue
-    
-print 'OK'

script/spider/weijingyun_pic_2_stationname.py

@@ -1,179 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-from django.db.models.fields import DateTimeField
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-
-# 防止https证书校验不正确
-import ssl
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class tiantianPort(Searchable):    
-    addr = StringField(default = '')
-    portId = StringField(default = '')
-    
-    meta = {
-        'collection': 'tiantian_port',
-        'db_alias': 'spider',
-        'unique_together': {'portId'}
-    }
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-# 拼接通用文字识别高精度url
-#
-rootdir = u'F:/蔚景云截图/'
-listFile = os.listdir(rootdir)  # 列出文件夹下所有的目录与文件
-resultList = []
-for i in range(0, len(listFile)):
-    print i
-    try:
-        path = os.path.join(rootdir, listFile[i])
-        f = open(path, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        access_token = token
-        request_url = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url, data=params, headers=headers)
-#         if response:
-#             print (response.json())
-    
-        # 解析返回结果
-        result_json = response.json()
-        
-        addr = ''
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            if len(text) < 7:
-                continue
-            if u'度起' in text or u'共' in text or 'km' in text or u'搜索' in text or u'闲' in text or u'VIP' in text or u'停车' in text or u'度' in text or '.' in text:
-                continue
-            
-            resultList.append(text)
-    except Exception:
-        continue
-    
-resultList = list(set(resultList))
-for result in resultList:
-    print result    
-
-# 打印文字
-print('OK')

script/spider/weitiandi.py

@@ -1,73 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class weitiandiDevice(Searchable):
-    meta = {
-        'collection': 'weitiandi_device',
-        'db_alias': 'spider',
-    }
-    
-# 根据二维码编号，获取设备更详细的信息
-logicalFormat1 = 'XC%04d'
-logicalFormat2 = 'XC%05d'
-
-url1 = 'http://wetiandi.com/app/index.php?i=1&j=1&c=entry&m=wdl_shopping&do=wepay&sn=GD1B%s'
-cookie1 = 'acw_tc=2f624a4616505087021958680e4bd61e40de02b46746403ae5406a5920172a; PHPSESSID=289df6867a76371820780cdbedce0397; PHPSESSID=289df6867a76371820780cdbedce0397'
-agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63060012)'
-#36000-80000
-newCount = 0
-for ii in range(20000,110000): 
-    try:
-        sn = 'GD1B%s' % ii
-        if weitiandiDevice.get_collection().find({'sn':sn}).count() >= 1:
-            print 'exist one',ii
-            continue
-        url = url1 % ii
-        response = requests.get(url,headers = {'Cookie':cookie1,'User-Agent':agent})
-        if '不存在' in response.content or '抱歉' in response.content or '异常' in response.content:
-            print 'offline',ii
-            continue
-        weiidIndex = response.content.find('var dev = ')
-        if not weiidIndex:
-            continue
-        endIndex = response.content.find(';',weiidIndex+11)
-        if not endIndex:
-            continue
-        devInfo = response.content[weiidIndex+10:endIndex]
-        devDict = json.loads(devInfo)
-        newCount += 1
-        print 'catch new one !',newCount
-        
-        devDict['spiderData'] = '0421'
-        weitiandiDevice.get_collection().update({'mac':devDict['mac']},{'$set':devDict},upsert = True)
-    except Exception,e:
-        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!',e
-        continue
-print('OK')

script/spider/weitiandiExport.py

@@ -1,146 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class weitiandiDevice(Searchable):
-    meta = {
-        'collection': 'weitiandi_device',
-        'db_alias': 'spider',
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-# ownerDict = {}
-# for shopInfo in pianyichongDevice.get_collection().find():
-#     if not shopInfo.has_key('manager'):
-#         continue
-#     if ownerDict.has_key(shopInfo['manager']):
-#         ownerDict[shopInfo['manager']] += 1
-#     else:
-#         ownerDict[shopInfo['manager']] = 1
-
-# records = []
-# ii = 0
-# for rcd in shankailaidianDevice.get_collection().find({}):
-#     ii += 1
-#     if ii % 100 == 0:
-#         print ii
-#     dataList = [
-#         (u'编号', str(rcd['vStationNo'])),
-#         (u'总收入', float(rcd['dTotalIncome'])),
-#         (u'地域ID', str(rcd['iAreaId'])),
-#         (u'businessType', str(rcd['iBusinessType'])),
-#         (u'iManagerId', str(rcd['iManagerId'])),
-#         (u'添加时间', str(rcd['tAddTime'])),
-#         (u'打开时间', str(rcd['tOpenTime'])),
-#         (u'地址', str(rcd['vAddress'])),
-#         (u'地域', str(rcd['vAreaName'])),
-#         (u'电话', rcd['vHelpMobile'] if rcd['vHelpMobile'] != '' else rcd['vTelephone']),
-# #         (u'充电站名称', str(rcd['vStationName'])),
-#     ]
-# 
-#     records.append(OrderedDict(dataList))
-# 
-# generate_excel_report('F:/shankailaidian1.xlsx', records,True)
-
-
-records = []
-owerDict = {}
-ii = 0
-
-# for rcd in weitiandiDevice.get_collection().find({'sn':{'$gte':'GD1B90000'}}):
-#     ii += 1
-#     if ii % 100 == 0:
-#         print ii
-#     dataList = [
-#         (u'编号', rcd['sn']),
-#         (u'联系方式', rcd['mobile']),
-#     ]
-# 
-#     records.append(OrderedDict(dataList))
-# generate_excel_report('F:/weitiandi_4G.xlsx', records,True)
-for rcd in weitiandiDevice.get_collection().find({'spiderData':'0421'}):
-    if rcd['mobile'] not in owerDict:
-        owerDict[rcd['mobile']] = 1
-    else:
-        owerDict[rcd['mobile']] += 1
-        
-for mobile,count in owerDict.items():
-    ii += 1
-    if ii % 100 == 0:
-        print ii
-    dataList = [
-        (u'联系方式', mobile),
-        (u'个数', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-generate_excel_report('F:/weitiandi_4G_count2.xlsx', records,True)
-print('OK')

script/spider/weitiandi_tel.py

@@ -1,60 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class weitiandiDevice(Searchable):
-    meta = {
-        'collection': 'weitiandi_device',
-        'db_alias': 'spider',
-    }
-    
-# 根据二维码编号，获取设备更详细的信息
-logicalFormat1 = 'XC%04d'
-logicalFormat2 = 'XC%05d'
-
-url1 = 'http://wetiandi.com/api/dev_info.php?do=customer_service&uid=%s'
-cookie1 = 'acw_tc=2f624a3316508485227543969e05cbdf3f8e34e8b34cb07b98a88d47257b6d; PHPSESSID=289df6867a76371820780cdbedce0397;PHPSESSID=289df6867a76371820780cdbedce0397'
-agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63060012)'
-#36000
-newCount = 0
-ii = 0
-for dev in weitiandiDevice.get_collection().find({'spiderData':'0421'}): 
-    try:
-        url = url1 % dev['weid']
-        response = requests.get(url,headers = {'Cookie':cookie1,'User-Agent':agent})
-        
-        mobile = json.loads(response.content).get('msg','')
-        weitiandiDevice.get_collection().update({'id':dev['id']},{'$set':{'mobile':mobile}},upsert = True)
-        ii += 1
-        if ii % 100 == 0:
-            print ii
-    except Exception,e:
-        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!',e
-        continue
-print('OK')

script/spider/xcy365.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class xcyDevice(Searchable):
-    devNo = StringField(default = '')
-    logicalCode = StringField(default = '') 
-    meta = {
-        'collection': 'xcy_device',
-        'db_alias': 'spider',
-    }
-    
-# 根据二维码编号，获取设备更详细的信息
-logicalFormat1 = 'XC%04d'
-logicalFormat2 = 'XC%05d'
-
-url1 = 'http://xcy365.com/gateway/agent/api/weixin/auth/device?deviceNo=%s&platformType=w'
-
-
-
-for ii in range(1590,15300):
-    try:
-        frm = logicalFormat1
-        if ii >= 10000:
-            frm = logicalFormat2    
-    
-        logicalCode = frm % ii
-        url = url1 % logicalCode
-        response = requests.get(url)
-        result = json.loads(response.content)
-        if result['code'] == 500:
-            print 'cant find devNo',logicalCode
-#             time.sleep(1)
-            continue
-        
-        redirect = result['data']['redirect']
-        listA = redirect.split('=')
-        devNo = listA[1]
-        
-        newDec = xcyDevice(devNo,logicalCode)
-        newDec.save()
-        
-#         time.sleep(1)
-    
-        print devNo,logicalCode
-    except Exception,e:
-        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!',e
-        continue
-print('OK')

script/spider/xcy365Service.py

@@ -1,58 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class xcyDevice(Searchable):
-    devNo = StringField(default = '')
-    logicalCode = StringField(default = '')
-    tel = StringField(default = '') 
-    name = StringField(default = '')
-    meta = {
-        'collection': 'xcy_device',
-        'db_alias': 'spider',
-    }
-    
-
-cookie1 = 'HDB_FPC=id=27f26b5f1f8bf7038f21647145062031:lv=1648020903231:ss=1648020437952:lsv=1647341426889:vs=1:spv=14'
-
-
-
-devNos = [obj.devNo for obj in xcyDevice.objects.all()]
-for devNo in devNos:
-    try:
-        url1 = 'http://xcy365.com/gateway/agent/api/device/%s/0' % devNo
-        refer = 'http://xcy365.com/h5/pages/charge/setting/bike_setting?deviceId=%s' % devNo
-        strhtml = requests.get(url1,headers = {'Cookie':cookie1,'Refer':refer,'TOKEN':'bca611e9-c5da-4ac0-b33d-36d73d6bd6f1'}).text
-        devData = json.loads(strhtml)
-        xcyDevice.get_collection().update({'devNo':devNo},{'$set':{'tel':devData['data']['repairsPhone'],'name':devData['data']['name']}})
-        print devNo,devData['data']['repairsPhone']
-    except Exception,e:
-        print '!!!!!!!!!!!!!!!!',e
-        continue 
-print('OK')

script/spider/xcy365Tel.py

@@ -1,99 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class xcyDevice(Searchable):
-    devNo = StringField(default = '')
-    logicalCode = StringField(default = '')
-    tel = StringField(default = '') 
-    name = StringField(default = '')
-    province = StringField(default = '')
-    city = StringField(default = '')
-    
-    meta = {
-        'collection': 'xcy_device',
-        'db_alias': 'spider',
-    }
-    
-
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    if tel[0] == '0':
-        print '',''
-    host = 'http://aiphone.market.alicloudapi.com'
-    path = '/ai_mobile_number_belong_to_china/v1'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'MOBILE_NUMBER=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        request.add_header('Content-Type', 'application/x-www-form-urlencoded; charset=UTF-8')
-        response = urllib2.urlopen(request)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            return result['ENTITY']['PROVINCE'],result['ENTITY']['CITY']
-    except Exception,e:
-        return '',''
-
-records = []
-telList = [obj.tel for obj in xcyDevice.objects.all() if obj.province=='' and obj.tel and obj.tel[0] != '0']
-telList = list(set(telList))
-
-for tel in telList:
-    p,c = get_tel_zone(tel)
-    print p,c,tel
-    xcyDevice.get_collection().update({'tel':tel},{'$set':{'province':p,'city':c}},multi=True)
-    
-for obj in xcyDevice.objects.all():
-    dataList = [
-        (u'logialCode', obj.logicalCode),        
-        (u'联系', obj.tel),
-        (u'名称', obj.name),
-        (u'省', obj.province),
-        (u'市', obj.city),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/xcy365.xlsx', records,True)
-
-print('OK')

script/spider/xiaozizhuang.py

@@ -1,68 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class xzzDevice(Searchable):
-    meta = {
-        'collection': 'xiaozizhuang',
-        'db_alias': 'spider',
-        'unique_together': {'qrId'}
-    }
-
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-url1 = 'https://trade.api3.sdaascloud.com/trade/device/status?qrId=%s&openId=ogDkt0_M-MX7kRgRuTG0xhf9eDnc&payType=WXPAY&latitude=30.480742&longitude=114.42102'
-devs = xzzDevice.get_collection().find()
-jj = 0
-for dev in devs:
-    try:
-        jj += 1
-        print(jj)
-        if dev.has_key('phone'):
-            continue
-        url = url1 % (dev['qrId'])
-        strhtml = requests.get(url,timeout = 15).text
-        result = json.loads(strhtml)
-        data = result['data']
-        xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':data},upsert = True)
-    except Exception,e:
-        continue
-    
-print('OK')

script/spider/xiaozizhuang_stat.py

@@ -1,95 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-# 统计U净用户的数据
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class xzzDevice(Searchable):
-    meta = {
-        'collection': 'xiaozizhuang',
-        'db_alias': 'spider',
-        'unique_together': {'qrId'}
-    }
-    
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-ownerDict = {}
-for shopInfo in xzzDevice.get_collection().find():
-    if not shopInfo.has_key('phone'):
-        continue
-    if ownerDict.has_key(shopInfo['phone']):
-        ownerDict[shopInfo['phone']] += 1
-    else:
-        ownerDict[shopInfo['phone']] = 1
-
-records = []
-for mobile,count in ownerDict.items():
-    zone = get_tel_zone(mobile)
-    print zone[0],zone[1],mobile,count
-    dataList = [
-        (u'省份', zone[0]),
-        (u'城市', zone[1]),
-        (u'通讯', mobile),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test2.xlsx', records,False)

script/spider/xingke.py

@@ -1,56 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-# 根据二维码编号，获取设备更详细的信息
-url1 = 'http://iot.xingkeele.com/mobile/personal/remote_socket_check'
-for ii in range(30000,40000):
-    data = {'socket_id':306996}
-    cookies = 'XKSESSION=tpki5ao5234kgqd31qob9lq5l6; mobile_user=V_Xjwavxt1864oRA_oY_'
-    
-    header = {'Accept': 'application/json, text/javascript, */*; q=0.01',
-              'Accept-Encoding': 'gzip, deflate',
-              'Accept-Language': 'zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7',
-              'Connection': 'keep-alive',
-              'Cookie': 'XKSESSION=tpki5ao5234kgqd31qob9lq5l6; mobile_user=V_Xjwavxt1864oRA_oY_',
-              'Host': 'iot.xingkeele.com',
-              'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Redmi Note 4X Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/77.0.3865.120 MQQBrowser/6.2 TBS/045513 Mobile Safari/537.36 MMWEBID/4983 MicroMessenger/8.0.1.1840(0x2800015B) Process/tools WeChat/arm64 Weixin NetType/WIFI Language/zh_CN ABI/arm64',
-              'Content-Length': '16',
-              'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
-              'Referer': 'http://iot.xingkeele.com/mobile/personal/charge_remote',
-              'Origin': 'http://iot.xingkeele.com',
-              
-              }
-    
-    strhtml = requests.post(url1,data=data,headers = header).text
-    result = json.loads(strhtml)
-    print result
-
-print('OK')

script/spider/xingxing_get_station_name_pic.py

@@ -1,61 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-import pyautogui
-
-# 首先输入访问设备的URL
-def send_url_for_dev(device_id):
-    time.sleep(1)
-#     x, y = pyautogui.position()
-#     print x,y
-    pyautogui.moveTo(1057,798)
-    pyautogui.click()
-    time.sleep(0.1)
-    pyautogui.hotkey('ctrlleft', 'a')
-    time.sleep(0.1)
-    pyautogui.press('backspace')
-    time.sleep(0.1)
-    pyautogui.typewrite(message='http://hzchaotiankj.com/weixin/payone?num=%s' % device_id)
-    time.sleep(0.1)
-    pyautogui.hotkey('altleft', 's')
-    time.sleep(0.1)
-    pyautogui.moveTo(1112,630)
-    pyautogui.click()
-    time.sleep(5)
-    pyautogui.moveTo(1260,129)
-    time.sleep(0.2)
-    pyautogui.click()
-
-# while True:
-#     print pyautogui.position()
-#     time.sleep(1)
-time.sleep(1)
-ii = 0
-while True:
-    pyautogui.moveTo(987,574)
-    file1 = pyautogui.screenshot(region=(766,220,412,575))
-    file1.save(u'Q:/友商信息/汽车桩/星星充电/站列表截图/%s.png' % int(time.time()*1000))
-    ii += 1
-    print ii
-    pyautogui.scroll(-400)
-    time.sleep(0.2)
-    
-print 'OK'

script/spider/xingxing_pic_2_stationname.py

@@ -1,183 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-from django.db.models.fields import DateTimeField
-
-import shutil
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-
-# 防止https证书校验不正确
-import ssl
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class xingxingStation(Searchable):    
-    name = StringField(default = '')
-    servicePhone = StringField(default = '')
-    company = StringField(default = '')
-    
-    
-    meta = {
-        'collection': 'xingxing_station',
-        'db_alias': 'spider',
-        'unique_together': {'name'}
-    }
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-# 拼接通用文字识别高精度url
-#
-rootdir = u'Q:/友商信息/汽车桩/星星充电/站列表截图'
-listFile = os.listdir(rootdir)  # 列出文件夹下所有的目录与文件
-resultList = []
-for i in range(0, len(listFile)):
-    print i
-    try:
-        path = os.path.join(rootdir, listFile[i])
-        f = open(path, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        access_token = token
-        request_url1 = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url1, data=params, headers=headers)
-#         if response:
-#             print (response.json())
-    
-        # 解析返回结果
-        result_json = response.json()
-        
-        needCopy = False 
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            
-            if u'他营' in text or u'联营' in text:
-                needCopy = True
-        
-        if needCopy:
-            print 'get one',i 
-            shutil.copyfile(rootdir+'/' + listFile[i], 'Q:/友商信息/汽车桩/星星充电/他营/%s' % listFile[i])
-    except Exception,e:
-        continue
-    
-resultList = list(set(resultList))
-for result in resultList:
-    print result    
-
-# 打印文字
-print('OK')

script/spider/xujiu.py

@@ -1,99 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-import pyautogui
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-def get_shops_info(lat,lng):
-    url = 'https://u.zhinengxiyifang.cn/api/Stores/findNearStoreInc'
-    payload = {"lat":lat,"lont":lng,"skip":0,"limit":10}
-#     url = 'https://www.baidu.com'
-#     payload = {}
-    proxies = {'http':'http://140.246.89.239:19382','https':'https://140.246.89.239:19382'}
-    strhtml = requests.post(url, json = payload,proxies = proxies,timeout = 15).text
-    result = json.loads(strhtml)
-    return result
-
-class xujiuDevice(Searchable):    
-    meta = {
-        'collection': 'xujiu_device',
-        'db_alias': 'spider',
-        'unique_together': {'device_id'}
-    }
-
-class spiderLog(Searchable):
-    device_id = StringField(default = '')
-    addedTime = DateTimeField(default = datetime.datetime.now())
-    meta = {
-        'collection': 'xujiu_spiderLog',
-        'db_alias': 'spider',
-        'unique_together': {'device_id'}
-    }
-    
-# 首先输入访问设备的URL
-def send_url_for_dev(device_id):
-    time.sleep(1)
-#     x, y = pyautogui.position()
-#     print x,y
-    pyautogui.moveTo(1057,798)
-    pyautogui.click()
-    time.sleep(0.1)
-    pyautogui.hotkey('ctrlleft', 'a')
-    time.sleep(0.1)
-    pyautogui.press('backspace')
-    time.sleep(0.1)
-    pyautogui.typewrite(message='http://hzchaotiankj.com/weixin/payone?num=%s' % device_id)
-    time.sleep(0.1)
-    pyautogui.hotkey('altleft', 's')
-    time.sleep(0.1)
-    pyautogui.moveTo(1112,630)
-    pyautogui.click()
-    time.sleep(5)
-    pyautogui.moveTo(1260,129)
-    time.sleep(0.2)
-    pyautogui.click()
-
-while True:
-    try:
-        devs = xujiuDevice.get_collection().find()
-        for dev in devs:
-            try:
-                if dev['device_id'] and dev['device_id'][0] == '0':
-                    continue
-                device_id = dev['device_id']
-                count =  spiderLog.objects.filter(device_id = device_id).count()
-                if count > 0:
-                    continue
-                
-                send_url_for_dev(device_id)
-                spiderLog(device_id = device_id).save()
-            except Exception,e:
-                continue
-    except Exception,e:
-        continue    
-print 'OK'

script/spider/xujiuExport.py

@@ -1,107 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class xujiu_device_tel(Searchable):
-    meta = {
-        'collection': 'xujiu_dev_user',
-        'db_alias': 'spider',
-        'unique_together': {'gtel'}
-    }
-
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-
-userDict = {}
-for user in xujiu_device_tel.get_collection().find():
-    if not user.has_key('manager_phone') or not user['manager_phone']:
-        continue
-    phone = user['manager_phone']
-    if userDict.has_key(phone):
-        userDict[phone] += 1
-    else:
-        userDict[phone] = 1
-        
-records = []
-for phone,count in userDict.items():
-    zone = get_tel_zone(phone)
-    dataList = [
-        (u'省份', zone[0]),
-        (u'城市', zone[1]),
-        (u'通讯', phone),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test4.xlsx', records,False)
-print('OK')

script/spider/xujiu_batch_dev.py

@@ -1,57 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-import pyautogui
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-
-class xujiuDevice(Searchable):    
-    meta = {
-        'collection': 'xujiu_device',
-        'db_alias': 'spider',
-        'unique_together': {'device_id'}
-    }
-
-rootdir = u'F:/爬虫相关/蓄久分析/all2'
-listFile = os.listdir(rootdir)  # 列出文件夹下所有的目录与文件
-for i in range(0, len(listFile)):
-    print i
-    try:
-        path = os.path.join(rootdir, listFile[i])
-        with open(path, 'rt') as f:
-            raw_data = ''.join(f.readlines())
-            data = json.loads(raw_data)
-            dataList = data['data']
-            for dev in dataList:
-                try:
-                    xujiuDevice.get_collection().update({'device_id':dev['device_id']},{'$set':dev},upsert = True)
-                except Exception,e:
-                    continue
-    except Exception,e:
-        continue
-print 'OK'

script/spider/yongxing_save_device_pic.py

@@ -1,269 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField,Document
-import simplejson as json
-import simplejson as json
-import base64
-import sys
-import json
-import base64
-import urllib2
-from urllib import quote_plus
-from urllib2 import urlopen
-from urllib2 import Request
-from urllib2 import URLError
-from urllib import urlencode
-
-# from django.db.models.fields import DateTimeField
-# 
-# 
-# os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-# 
-# from script.base import init_env
-# 
-# init_env(interactive = False)
-
-import pyautogui
-import qrcode
-
-IS_PY3 = sys.version_info.major == 3
-if IS_PY3:
-    from urllib.request import urlopen
-    from urllib.request import Request
-    from urllib.error import URLError
-    from urllib.parse import urlencode
-    from urllib.parse import quote_plus
-else:
-    import urllib2
-    from urllib import quote_plus
-    from urllib2 import urlopen
-    from urllib2 import Request
-    from urllib2 import URLError
-    from urllib import urlencode
-
-# 防止https证书校验不正确
-import ssl
-
-# while True:
-#     a = pyautogui.getWindowsWithTitle(u'图片查看')
-#     time.sleep(1)
-    
-ssl._create_default_https_context = ssl._create_unverified_context
-
-# 利用百度APP，直接解析截图中的地址，以及端口编号。
-API_KEY = 'OVcN78LP40CBEwWk5REF2Hyu'
-SECRET_KEY = 'a7luZBdbzjsfU9oE2GD3yPeTBgPty03t'
-OCR_URL = "https://aip.baidubce.com/rest/2.0/ocr/v1/accurate_basic"
-TOKEN_URL = 'https://aip.baidubce.com/oauth/2.0/token'
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class yongxing(Document):    
-    devNo = StringField(default = '')
-    phone = StringField(default = '')
-
-    meta = {
-        'collection': 'yongxing',
-        'db_alias': 'spider',
-        'unique_together': {'devNo'}
-    }
-    
-def fetch_token():
-    params = {'grant_type': 'client_credentials',
-              'client_id': API_KEY,
-              'client_secret': SECRET_KEY}
-    post_data = urlencode(params)
-    if (IS_PY3):
-        post_data = post_data.encode('utf-8')
-    req = Request(TOKEN_URL, post_data)
-    try:
-        f = urlopen(req, timeout=5)
-        result_str = f.read()
-    except URLError as err:
-        print(err)
-    
-    if (IS_PY3):
-        result_str = result_str.decode()
-        
-    result = json.loads(result_str)
-
-    if ('access_token' in result.keys() and 'scope' in result.keys()):
-        if not 'brain_all_scope' in result['scope'].split(' '):
-            print ('please ensure has check the  ability')
-            exit()
-        return result['access_token']
-    else:
-        print ('please overwrite the correct API_KEY and SECRET_KEY')
-        exit()
-        
-request_url = "https://aip.baidubce.com/rest/2.0/ocr/v1/general_basic" # accurate_basic
-
-def read_file(image_path):
-    f = None
-    try:
-        f = open(image_path, 'rb')
-        return f.read()
-    except:
-        print('read image file fail')
-        return None
-    finally:
-        if f:
-            f.close()
-            
-def request(url, data):
-    req = Request(url, data.encode('utf-8'))
-    has_error = False
-    try:
-        f = urlopen(req)
-        result_str = f.read()
-        if (IS_PY3):
-            result_str = result_str.decode()
-        return result_str
-    except  URLError as err:
-        print(err)
-        
-# 二进制方式打开图片文件
-# 获取access token
-token = fetch_token()
-# 拼接通用文字识别高精度url
-#
-def get_phone(devNo,path):
-    try:
-        f = open(path, 'rb')
-        img = base64.b64encode(f.read())
-        if f:
-            f.close()
-        # 调用文字识别服务
-        params = {"image":img}
-        request_url1 = request_url + "?access_token=" + token
-        headers = {'content-type': 'application/x-www-form-urlencoded'}
-        response = requests.post(request_url1, data=params, headers=headers)
-
-        # 解析返回结果
-        result_json = response.json()
-        
-        for words_result in result_json["words_result"]:
-            text = words_result["words"]
-            
-            if u'打' in text and len(text)>7:
-                phone = text.replace(u'打',"")
-                print u'获取到一条电话！！',devNo,phone
-                try:
-                    yongxing(devNo,phone).save()
-                    break
-                except Exception,e:
-                    break
-        
-    except Exception,e:
-        print u"解析图片报错了！！！！",e
-        
-    
-for ii in range(0,84217):
-    jj = 84217 - ii
-    print jj
-    qrStr = 'https://a.nblinks.cn/c?d=1000%04d' % jj
-    img =qrcode.make(qrStr)
-    filePath = "Q:/temp/%s" % jj
-    img.save(filePath)
-#     time.sleep(0.2)
-       
-    weixin = pyautogui.getWindowsWithTitle(u'微信')[0]
-    if not weixin:
-        break
-    weixin.activate()
-    
-    # 点击文件按钮
-    pyautogui.click(x=761, y=777)
-    time.sleep(0.2)
-       
-    # 输入二维码文件名称
-    pyautogui.click(x=1083, y=928)
-    pyautogui.write("%s" % jj)
-    time.sleep(0.2)
-       
-    # 选择该文件,传到编辑框
-    with pyautogui.hold('alt'):
-        pyautogui.press('o')
-    time.sleep(0.1)
-    
-    # 双击打开二维码
-    pyautogui.doubleClick(x=791, y=816)
-    time.sleep(1)
-    
-    # 右键打开
-    pyautogui.rightClick(x=983,y=468)
-    time.sleep(0.1)
-    # 点击识别二维码
-    pyautogui.click(x=1084,y=553)
-     
-    # 等待小程序启动界面出来.如果
-    time.sleep(1.7)
-    title = pyautogui.getActiveWindowTitle()
-    timeCount = 0
-    if title != u'富连e充':
-        print u" 无法拉起来小程序"
-        time.sleep(1800)
-    
-    # 截图
-    file1 = pyautogui.screenshot(region=(758,113,402,765))
-    picPath = u'Q:/友商信息/涌鑫/小程序截图/' + '%s.png' % int(time.time()*1000)
-    file1.save(picPath)
-    get_phone('1000%04d' % jj, picPath)
-    
-    pyautogui
-    #  加按几次esc，用于消掉一些提示
-    pyautogui.press('esc')
-    time.sleep(0.5)
-    
-#     # 点击关闭小程序
-#     pyautogui.click(x=1135, y=124)
-#     time.sleep(0.1)
-#     
-#     # 关闭截图
-#     pyautogui.click(x=983,y=468)
-#     time.sleep(0.2)
-#     pyautogui.press('esc')
-#     time.sleep(0.2)
-    
-    # 删除截图
-#     pyautogui.click(x=791, y=816)
-    weixin = pyautogui.getWindowsWithTitle(u'微信')[0]
-    weixin.activate()
-    
-    pyautogui.click(x=1413, y=800)
-    time.sleep(0.5)
-    
-    with pyautogui.hold('ctrl'):
-        pyautogui.press('a')
-    pyautogui.press('delete')
-#     time.sleep(0.1)
-# 
-# 
-# while True:
-#     a= auto.GetFocusedControl()
-#     time.sleep(2)
-#     print a
-#     print a.Name
-#     time.sleep(2)
-
-
-# while True:
-#     pyautogui.moveTo(987,574)
-#     file1 = pyautogui.screenshot(region=(766,220,412,575))
-#     file1.save(u'Q:/友商信息/汽车桩/星星充电/站列表截图/%s.png' % int(time.time()*1000))
-#     ii += 1
-#     print ii
-#     pyautogui.scroll(-400)
-#     time.sleep(0.2)
-    
-print 'OK'

script/spider/youdianchong(dahan).py

@@ -1,69 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class youdianchongDevice(Searchable):
-    meta = {
-        'collection': 'youdianchong_device',
-        'db_alias': 'spider',
-        'unique_together': {'id'}
-    }
-
-# 根据二维码编号，获取设备更详细的信息
-url = 'http://ydc.youdianchong.com:8087/newCloseCharger?longitude=%s&latitude=%s'
-govs = Goverment.get_collection().find()
-for gov in govs:
-    try:
-        print gov['name']
-        url1 = url % (gov['lng'],gov['lat'])
-        strhtml = requests.get(url1,headers = {'openId':'ozk7Zv4WC5akrk9Lvrm78jggwHk4'}).text
-        result = json.loads(strhtml)
-        if not result.has_key('data'):
-            continue
-        if len(result['data'])== 0:
-            continue
-        print len(result['data'])
-        for dev in result['data']:
-            youdianchongDevice.get_collection().update({'id':dev['id']},{'$set':dev},upsert = True)
-    except Exception,e:
-        continue
-print('OK')

script/spider/youdianchongExport.py

@@ -1,107 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class pianyichongDevice(Searchable):
-    meta = {
-        'collection': 'pianyichong_device',
-        'db_alias': 'spider',
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-ownerDict = {}
-for shopInfo in pianyichongDevice.get_collection().find():
-    if not shopInfo.has_key('manager'):
-        continue
-    if ownerDict.has_key(shopInfo['manager']):
-        ownerDict[shopInfo['manager']] += 1
-    else:
-        ownerDict[shopInfo['manager']] = 1
-
-records = []
-for shop,count in ownerDict.items():
-#     zone = get_tel_zone(mobile)
-#     print zone[0],zone[1],mobile,count
-#     city,contact = '',''
-#     tempList = shop.split(' ')
-#     if len(tempList) > 2:
-#         city,contact = tempList[0],tempList[1]
-    dataList = [
-        (u'通讯', shop),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test7.xlsx', records,False)
-print('OK')

script/spider/yunchongExport.py

@@ -1,108 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class yunxiangDevice(Searchable):
-    meta = {
-        'collection': 'yunxiang_device',
-        'db_alias': 'spider',
-        'unique_together': {'gtel'}
-    }
-
-class yunxiangUser(Searchable):
-    meta = {
-        'collection': 'yunxiang_user',
-        'db_alias': 'spider',
-        'unique_together': {'uUser'}
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-records = []
-for user in yunxiangUser.get_collection().find():
-    if not user.has_key('utel'):
-        continue
-    
-    zone = get_tel_zone(user['utel'])
-    dataList = [
-        (u'省份', zone[0]),
-        (u'城市', zone[1]),
-        (u'通讯', user.get('utel','')),
-        (u'数量', user.get('devCount',0)),
-        (u'微信昵称', user.get('wxnc','')),
-        (u'姓名', user.get('uxm','')),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test3.xlsx', records,False)
-print('OK')

script/spider/yunchongTel.py

@@ -1,87 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class yunxiangDevice(Searchable):
-    meta = {
-        'collection': 'yunxiang_device',
-        'db_alias': 'spider',
-        'unique_together': {'gtel'}
-    }
-
-class yunxiangUser(Searchable):
-    meta = {
-        'collection': 'yunxiang_user',
-        'db_alias': 'spider',
-        'unique_together': {'uUser'}
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-usersDict = {}
-ii = 0
-for dev  in yunxiangDevice.get_collection().find():
-    print ii
-    ii += 1
-    if not dev.has_key('uUser'):
-        continue
-    if usersDict.has_key(dev['uUser']):
-        usersDict[dev['uUser']] += 1
-    else:
-        usersDict[dev['uUser']] = 1
-
-url1 = 'http://cdz.gpsserver.cn/USearch'
-
-ii = 0
-for user,count in usersDict.items():
-    print ii
-    ii += 1
-    try:
-        data = {'uUser':user,'sjtime':1614939644497}
-        strhtml = requests.post(url1,data=data).text
-        result = json.loads(strhtml)
-        if len(result) == 0:
-            continue
-        data = result[0]
-    
-        data.update({'devCount':count})
-        yunxiangUser.get_collection().update({'uUser':data['uUser']},{'$set':data},upsert=True)
-    except Exception,e:
-        continue
-print('OK')

script/spider/yunxiangzhichong.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class yunxiangDevice(Searchable):
-    meta = {
-        'collection': 'yunxiang_device',
-        'db_alias': 'spider',
-        'unique_together': {'gtel'}
-    }
-
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-url1 = 'http://cdz.gpsserver.cn/ChargeCar?gtel=680000%05d'
-for ii in range(20000):
-    try:
-        print ii
-        url = url1 % ii
-        strhtml = requests.get(url,timeout = 15).text
-        result = json.loads(strhtml)
-        if len(result) == 0:
-            continue
-        data = result[0]
-        yunxiangDevice.get_collection().update({'gtel':data['gtel']},{'$set':data},upsert=True)
-    except Exception,e:
-        continue
-print('OK')

script/spider/yunyichong.py

@@ -1,83 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class yunyichongDevice(Searchable):
-    uuid = StringField(default = '')
-    meta = {
-        'collection': 'yunyichong_device',
-        'db_alias': 'spider',
-        'unique_together': {'uuid'}
-    }
-
-class errorLog(Searchable):
-    meta = {
-        'collection': 'wanzhuang_err_location',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-# 根据二维码编号，获取设备更详细的信息
-url = 'https://wx-yyc.dianxiaomei.com.cn/ui/nearby?lat=%s&lng=%s&city='
-cookie1 = 'PHPSESSID=0vcenbi6pgu6stup1brlelmo42; yyc_auth_v3=HNfsUaXyxVUL68umVR23r1JDNJd%2BwEEMXCH1oSMEbg%2FE08uwJGjseQE%2F7QigXiBx; uid=19599418;'
-govs = Goverment.get_collection().find()
-ii = 0
-for gov in govs[19970:]:
-    try:
-        ii += 1
-        print ii,gov['name']
-        if gov['name'][-2:] != u'政府':
-            continue 
-        url1 = url % (gov['lat'],gov['lng'])
-        cookies = cookie1 + '_lat=%s; _lng=%s;' % (gov['lat'],gov['lng']) + 'j_w_city=%E6%AD%A6%E6%B1%89%E5%B8%82'
-        strhtml = str(requests.get(url1,verify=False,headers = {'Cookie':cookies}).text)
-        startIndex = strhtml.find('device:[')
-        endIndex = strhtml.find(']',startIndex) 
-        if not startIndex or not endIndex:
-            continue
-        devListStr = strhtml[startIndex+7:endIndex+1]
-        devListDict = json.loads('{"device":%s}' % devListStr)
-        
-        for dev in devListDict['device']:
-            yunyichongDevice.get_collection().update({'uuid':dev['uuid']},{'$set':dev},upsert = True)
-    except Exception,e:
-        continue
-print('OK')

script/spider/yunyichongExport.py

@@ -1,111 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import urllib, urllib2, sys
-import ssl
-from django.db.models.fields import DateTimeField
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class yunyichongDevice(Searchable):
-    uuid = StringField(default = '')
-    meta = {
-        'collection': 'yunyichong_device',
-        'db_alias': 'spider',
-        'unique_together': {'uuid'}
-    }
-# 先把所有的设备二维码编号爬下来
-# page=0
-# while True:
-#     url = 'https://trade.api3.sdaascloud.com/trade/nearbyChargers?key=&page=%s&size=10&latitude=30.48029&longitude=114.42073&raidus=3000000000' % page
-#     strhtml = requests.get(url,timeout = 15).text
-#     result = json.loads(strhtml)
-#     if result.has_key('data') and result['data'].has_key('content') and len(result['data']['content']) == 0:
-#         break
-#     for dev in result['data']['content']:
-#         xzzDevice.get_collection().update({'qrId':dev['qrId']},{'$set':dev},upsert = True)
-#     page += 1
-
-# 根据二维码编号，获取设备更详细的信息
-def get_tel_zone(tel):
-    if not tel or len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-ownerDict = {}
-for shopInfo in yunyichongDevice.get_collection().find():
-    if not shopInfo.has_key('tel'):
-        continue
-    if ownerDict.has_key(shopInfo['tel']):
-        ownerDict[shopInfo['tel']] += 1
-    else:
-        ownerDict[shopInfo['tel']] = 1
-
-records = []
-for mobile,count in ownerDict.items():
-    zone = get_tel_zone(mobile)
-    print zone[0],zone[1],mobile,count
-#     city,contact = '',''
-#     tempList = shop.split(' ')
-#     if len(tempList) > 2:
-#         city,contact = tempList[0],tempList[1]
-    dataList = [
-        (u'省份', zone[0]),
-        (u'市', zone[1]),
-        (u'通讯', mobile),
-        (u'数量', count),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/test9.xlsx', records,False)
-print('OK')

script/spider/yunyichongTel.py

@@ -1,73 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '116.62.228.194',
-                    port = 27017,
-                    username = 'dba',
-                    password = 'dayuan@2020..',
-                    authentication_source = 'admin')
-
-class Goverment(Searchable):
-    province = StringField(default = "")
-    city = StringField(default = "")
-    name = StringField(default = '')
-    lat = StringField(default = '')
-    lng = StringField(default = '') 
-    
-    meta = {
-        'collection': 'Goverment',
-        'db_alias': 'spider',
-        'unique_together': {'lat', 'lng'}
-    }
-    
-class yunyichongDevice(Searchable):
-    uuid = StringField(default = '')
-    meta = {
-        'collection': 'yunyichong_device',
-        'db_alias': 'spider',
-        'unique_together': {'uuid'}
-    }
-
-# 根据二维码编号，获取设备更详细的信息
-url = 'https://wx-yyc.dianxiaomei.com.cn/ui/portlist?uuid=%s'
-cookie1 = 'PHPSESSID=0vcenbi6pgu6stup1brlelmo42; yyc_auth_v3=HNfsUaXyxVUL68umVR23r1JDNJd%2BwEEMXCH1oSMEbg%2FE08uwJGjseQE%2F7QigXiBx;' 
-devs = yunyichongDevice.get_collection().find()
-ii = 0
-for dev in devs:
-    try:
-        ii += 1
-        print ii
-        url1 = url % (dev['uuid'])
-        tempCookie = 'uid=%s;' % dev['uuid']
-        cookies = cookie1 + tempCookie
-        strhtml = str(requests.get(url1,verify=False,headers = {'Cookie':cookies}).text)
-        startIndex = strhtml.find('tel:')
-        endIndex = strhtml.find('"',startIndex) 
-        if not startIndex or not endIndex:
-            continue
-        tel = strhtml[startIndex+4:endIndex]
-        yunyichongDevice.get_collection().update({'uuid':dev['uuid']},{'$set':{'tel':tel}})
-    except Exception,e:
-        continue
-print('OK')

script/spider/zhiyichong.py

@@ -1,61 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class zhiyichongSeller(Searchable):
-    meta = {
-        'collection': 'zhiyichong_seller',
-        'db_alias': 'spider',
-    }
-    
-# 根据二维码编号，获取设备更详细的信息
-logicalFormat1 = 'XC%04d'
-logicalFormat2 = 'XC%05d'
-
-url1 = 'https://wx.isfdz.com/users/getInfo?id=1703433&srcId=200&sellerId=%s'
-# cookie1 = 'PHPSESSID=289df6867a76371820780cdbedce0397; PHPSESSID=289df6867a76371820780cdbedce0397; acw_tc=76b20ff916492062843861607e6040a09702c48cd1884b1201c1389a7b9c14'
-agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63060012)'
-#36000-80000
-newCount = 0
-ii = 0
-for ii in range(3000): 
-    try:
-        url = url1 % ii
-        response = requests.get(url)
-        
-        devDict = json.loads(response.text)
-        if 'operatorPhone' not in devDict['data']:
-            print 'over -----------------------' 
-            continue
-        zhiyichongSeller.get_collection().update({'operatorPhone':devDict['data']['operatorPhone']},{'$set':devDict['data']},upsert = True)
-        print ii
-    except Exception,e:
-        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!',e
-        continue
-print('OK')

script/spider/zhiyichong_dev_count.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-
-from django.db.models.fields import DateTimeField
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-class zhiyichongDevice(Searchable):
-    meta = {
-        'collection': 'zhiyichong_dev',
-        'db_alias': 'spider',
-    }
-    
-# 根据二维码编号，获取设备更详细的信息
-
-url1 = 'https://wx.isfdz.com/wxConfig/getUserMoney'
-# cookie1 = 'PHPSESSID=289df6867a76371820780cdbedce0397; PHPSESSID=289df6867a76371820780cdbedce0397; acw_tc=76b20ff916492062843861607e6040a09702c48cd1884b1201c1389a7b9c14'
-agent = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36 NetType/WIFI MicroMessenger/7.0.20.1781(0x6700143B) WindowsWechat(0x63060012)'
-#36000-80000
-newCount = 0
-ii = 0
-for ii in range(60000,70000): 
-    try:
-        print ii
-        
-        url = url1
-        
-        data = {"code":"4D000%05d"%ii,"userId":1703433}
-        response = requests.post(url,json=data)
-        
-        devDict = json.loads(response.text)
-        if (devDict['code'] == 500) or ('mobile' not in devDict['data']) or (devDict['data']['mobile']==''):
-            print 'not exist device -----------------------' 
-            continue
-        devDict['data'].update({'devNo':"4D000%05d"%ii})
-        zhiyichongDevice.get_collection().update({'devNo':"4D000%05d"%ii},{'$set':devDict['data']},upsert = True)
-        print 'catch new one now......'
-    except Exception,e:
-        print '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!',e
-        continue
-print('OK')

script/spider/zhiyichong_export.py

@@ -1,79 +0,0 @@
-# -*- coding: utf-8 -*-
-# !/usr/bin/env python
-
-import os, sys,time,datetime
-import urllib, urllib2, sys
-import requests
-from mongoengine import register_connection, PointField, DynamicDocument, StringField
-import simplejson as json
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")
-
-from script.base import init_env
-
-init_env(interactive = False)
-
-from apps.web.core.db import Searchable
-from apps.web.core.utils import generate_excel_report
-
-register_connection(alias = 'spider',
-                    name = 'spider',
-                    host = '211.159.224.10',
-                    port = 27119,
-                    username = 'service',
-                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
-                    authentication_source = 'admin')
-
-
-class zhiyichongSeller(Searchable):
-    meta = {
-        'collection': 'zhiyichong_seller',
-        'db_alias': 'spider',
-    }
-    
-class zhiyichongDevice(Searchable):
-    meta = {
-        'collection': 'zhiyichong_dev',
-        'db_alias': 'spider',
-    }
-
-ii = 0
-records = []
-for rcd in zhiyichongSeller.get_collection().find({}):
-    ii += 1
-    if ii % 100 == 0:
-        print ii
-    dataList = [
-        (u'联系方式', rcd['operatorPhone']),
-    ]
- 
-    records.append(OrderedDict(dataList))
-    
-generate_excel_report('F:/zhiyichong_operator.xlsx', records,True)
-
-
-ii = 0
-records = []
-ownerDict = {}
-for rcd in zhiyichongDevice.get_collection().find({}):
-    if rcd['mobile'] not in ownerDict:
-        ownerDict[rcd['mobile']] = 1
-    else:
-        ownerDict[rcd['mobile']] += 1
-
-
-for mobile,count in ownerDict.items():
-    dataList = [
-        (u'联系方式', mobile),
-        (u'数目', count),
-    ]
- 
-    records.append(OrderedDict(dataList))    
-generate_excel_report('F:/zhiyichong_operator_count.xlsx', records,True)
-print('OK')

script/spider_data/dealer_stats.py

@@ -1,57 +0,0 @@
-# -*- coding: utf-8 -*-
-#!/usr/bin/env python
-
-import os
-
-import sys
-import datetime
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.update({"DJANGO_SETTINGS_MODULE": "configs.production"})
-import django
-django.setup()
-
-from apilib.utils_datetime import to_datetime
-
-from apps.web.constant import Const
-from apps.web.device.models import Device
-from apps.web.dealer.models import DealerRechargeRecord
-
-# 统计东郡的经销商数据，按照设备量的大小，从高统计
-import pymongo
-client=pymongo.MongoClient(host='211.159.224.10',port=27119,username = 'dba',password = 'dayuan@2020..')  #连接本地的服务端
-db=client.spider  #指定操作的数据库
-collection=db.dongjun_devices  #指定操作的集合
-
-# 遍历设备，并登记到经销商
-dealerDict = {}
-
-devs = collection.find({'createDate':{'$gte':1596211200000}})
-devList = []
-for dev in devs:
-    devList.append(dev)
-
-for dev in devList:
-    tel = dev.get('contactMobilePhone','')  
-    if tel not in dealerDict:
-        dealerDict[tel] = {'count':1,'cityName':dev.get('cityName',''),'typeName':dev.get('equipTypeName',''),'name':dev.get('name','')}
-    else:
-        dealerDict[tel]['count'] += 1
-
-
-dealerList = [(k,v['count'],v['cityName'],v['typeName'],v['name']) for k,v in dealerDict.items()]
-def cmp_xy(x,y):
-    if x[1] < y[1]:
-        return -1
-    elif x[1] == y[1]:
-        return 0
-    return 1
-
-dealerList.sort(cmp=cmp_xy, reverse=True)
-
-for dealer in dealerList[0:100000]:
-    print dealer[0],dealer[1],dealer[2],dealer[3],dealer[4]
-    

script/spider_data/device_info.py

@@ -1,39 +0,0 @@
-# -*- coding: utf-8 -*-
-#!/usr/bin/env python
-
-import os
-
-import sys
-import datetime
-
-
-PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
-sys.path.insert(0, PROJECT_ROOT)
-
-os.environ.update({"DJANGO_SETTINGS_MODULE": "configs.production"})
-import django
-django.setup()
-
-from apilib.utils_datetime import to_datetime
-
-from apps.web.constant import Const
-from apps.web.device.models import Device
-from apps.web.dealer.models import DealerRechargeRecord
-
-# 统计东郡的经销商数据，按照设备量的大小，从高统计
-import pymongo
-client=pymongo.MongoClient(host='localhost',port=27017,username = 'dba',password = 'dayuan@2020..')  #连接本地的服务端
-db=client.spider  #指定操作的数据库
-collection=db.dongjun_devices  #指定操作的集合
-
-# 遍历设备，并登记到经销商
-dealerDict = {}
-
-devs = collection.find({})
-devList = []
-for dev in devs:
-    devList.append(dev)
-
-for dev in devList:
-    print dev.get('contactMobilePhone','') ,dev.get('cityName',''),dev.get('equipTypeName',''),dev.get('name',''),dev.get('address',''),dev.get('areaName',''),dev.get('contactUser',''),dev.get('provinceName','')
-

script/spider_data/lyy_dealer_stats.py

@@ -1,115 +0,0 @@
-# -*- coding: utf-8 -*-
-#!/usr/bin/env python
-
-import os,json
-import urllib, urllib2, sys
-import ssl
-import xlrd
-from xlrd import xldate_as_tuple
-from collections import OrderedDict
-from apps.web.core.utils import generate_excel_report
-# 统计乐摇摇的经销商数据，按照设备量的大小，从高统计 https://m.leyaoyao.com/customer/message/t/81202242
-
-rootdir = u'F:/Temp/lyy'
-
-# 查询手机归属地的代码
-def get_tel_zone(tel):
-    if len(tel) <= 7:
-        return '',''
-    tel = tel.replace('-','')
-    tel = tel.replace(' ','')
-    print tel
-    host = 'https://api04.aliyun.venuscn.com'
-    path = '/mobile'
-    method = 'GET'
-    appcode = '8296fcd952e34713ba91fbdceb13e915'
-    querys = 'mobile=%s' % tel
-    bodys = {}
-    url = host + path + '?' + querys
-    try:
-        request = urllib2.Request(url)
-        request.add_header('Authorization', 'APPCODE ' + appcode)
-        ctx = ssl.create_default_context()
-        ctx.check_hostname = False
-        ctx.verify_mode = ssl.CERT_NONE
-        response = urllib2.urlopen(request, timeout = 15, context=ctx)
-        content = response.read()
-        if (content):
-            result = json.loads(content)
-            if result['msg'] != 'success':
-                return '', ''
-            return result['data']['prov'],result['data']['city']
-    except Exception,e:
-        return '',''
-    
-# 遍历设备，并登记到经销商
-
-dealerDict = {}
-allCount = 0
-listFile = os.listdir(rootdir)  # 列出文件夹下所有的目录与文件
-for i in range(0, len(listFile)):
-    path = os.path.join(rootdir, listFile[i])
-    try:
-        book = xlrd.open_workbook(path)
-        sheet = book.sheet_by_name("sheet1")
-    except Exception,e:
-        print 'open excel file error =%s' % e
-        continue
-    
-    nrows = sheet.nrows
-    rows = []
-    for i in range(1, nrows):
-        row = sheet.row_values(i)
-        allCount += 1
-        try:
-            tel = row[1]
-            if tel not in dealerDict:
-                dealerDict[tel] = {'count':1,'name':row[3],'devList':[row[0]]}
-            else:
-                dealerDict[tel]['count'] += 1
-                if len(dealerDict[tel]['devList']) < 10:
-                    dealerDict[tel]['devList'].append(str(row[0]))
-                    
-        except Exception,e:
-            print 'some error when update e=%s' % (e)
-
-dealerList = []        
-for k,v in dealerDict.items():
-    tempStr = ''
-    try:
-        tempStr = ','.join(v['devList'])
-    except Exception,e:
-        tempStr = ''
-    dealerList.append((k,v['count'],v['name'],tempStr))
-    
-def cmp_xy(x,y):
-    if x[1] < y[1]:
-        return -1
-    elif x[1] == y[1]:
-        return 0
-    return 1
-
-dealerList.sort(cmp=cmp_xy, reverse=True)
-
-records = []
-for dealer in dealerList:
-    try:
-        print u'电话：%s,数量：%s,设备名称：%s,设备编号举例：%s' % (dealer[0],dealer[1],dealer[2],dealer[3])
-    except Exception,e:
-        continue
-    
-    province,city = get_tel_zone(dealer[0])
-    dataList = [
-        (u'电话', dealer[0]),
-        (u'数量', dealer[1]),
-        (u'设备名称', dealer[2]),
-        (u'编号举例', dealer[3]),
-        (u'手机归属省份', province),
-        (u'手机归属城市', city),
-    ]
-
-    records.append(OrderedDict(dataList))
-
-generate_excel_report('F:/Temp/lyy/202009.xlsx', records, localSave = True)
-    
-print u'共计：%s' % allCount