manyi/script/spider/get_charger_station.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os, sys
import urllib
import requests
from mongoengine import register_connection, PointField, DynamicDocument, StringField
import simplejson as json
from django.db.models.fields import FloatField


PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing")

from script.base import init_env

init_env(interactive = False)


# 获取全国的大学和学院的地理位置信息
# strhtml = requests.post(url, json = payload,timeout = 15).text
# result = json.loads(strhtml)

regCodes = [33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,731,770,789,792,1214,1215,1216,1217,1218,1277,1293,1498,1515,1641,1642,1643,1644,1713,2031,2032,2033,2358,2359,2634,2654,2734,2757,2758]
# regCodes = [124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,731,770,789,792,1214,1215,1216,1217,1218,1277,1293,1498,1515,1641,1642,1643,1644,1713,2031,2032,2033,2358,2359,2634,2654,2734,2757,2758]
#regCodes = [169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,731,770,789,792,1214,1215,1216,1217,1218,1277,1293,1498,1515,1641,1642,1643,1644,1713,2031,2032,2033,2358,2359,2634,2654,2734,2757,2758]
#regCodes = [218]
register_connection(alias = 'spider',
                    name = 'spider',
                    host = '211.159.224.10',
                    port = 27119,
                    username = 'service',
                    password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5',
                    authentication_source = 'admin')


class ChargeStation(DynamicDocument):
    province = StringField(default = "")
    city = StringField(default = "")
    name = StringField(default = '')
    address = StringField(default = '')
    telephone = StringField(default = '')
    lat = StringField(default = '')
    lng = StringField(default = '')
    
    meta = {
        'collection': 'ChargeStation',
        'db_alias': 'spider',
        'unique_together': {'lat', 'lng'}
    }
    
    
def get_page(regCode,pageNum):
    baiduUrl = 'http://api.map.baidu.com/place/v2/search?query=充电桩&page_num=%s&region=%s&output=json&ak=3RVEkGkdaiSnVHn0dT5EGZu1RngcFlyP' % (pageNum,regCode)
    return json.loads(requests.get(baiduUrl,timeout=15).text)

def upsert_gov(result):
    for info in result['results']:
        if 'location' not in info:
            continue
        newSch = ChargeStation(province = info.get('province',''),city = info.get('city',''),name = info.get('name',''),address = info.get('address',''),telephone=info.get('telephone',''),lat = str(info['location']['lat']),lng = str(info['location']['lng']))
        
        try:
            newSch.save()
        except Exception,e:
            continue

pageNum = 0
for reg in regCodes:
    print reg
    getCount = 0
    pageNum = 0
    result = get_page(reg,pageNum)
    getCount += len(result['results'])
    upsert_gov(result)
    while getCount < result['total']:
        pageNum += 1
        result = get_page(reg,pageNum)
        upsert_gov(result)
        getCount += len(result['results'])
    
    print reg,getCount
    
print 'ok'