# -*- coding: utf-8 -*- # !/usr/bin/env python import os, sys import urllib import requests from mongoengine import register_connection, PointField, DynamicDocument, StringField import simplejson as json from django.db.models.fields import FloatField PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..') sys.path.insert(0, PROJECT_ROOT) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "configs.testing") from script.base import init_env init_env(interactive = False) # 获取全国的大学和学院的地理位置信息 # strhtml = requests.post(url, json = payload,timeout = 15).text # result = json.loads(strhtml) regCodes = [33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,731,770,789,792,1214,1215,1216,1217,1218,1277,1293,1498,1515,1641,1642,1643,1644,1713,2031,2032,2033,2358,2359,2634,2654,2734,2757,2758] # regCodes = [124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,731,770,789,792,1214,1215,1216,1217,1218,1277,1293,1498,1515,1641,1642,1643,1644,1713,2031,2032,2033,2358,2359,2634,2654,2734,2757,2758] #regCodes = [169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,731,770,789,792,1214,1215,1216,1217,1218,1277,1293,1498,1515,1641,1642,1643,1644,1713,2031,2032,2033,2358,2359,2634,2654,2734,2757,2758] #regCodes = [218] register_connection(alias = 'spider', name = 'spider', host = '211.159.224.10', port = 27119, username = 'service', password = 'oOzjoQcO5DyyiN97AY0NpzJ6vztjNpx5', authentication_source = 'admin') class ChargeStation(DynamicDocument): province = StringField(default = "") city = StringField(default = "") name = StringField(default = '') address = StringField(default = '') telephone = StringField(default = '') lat = StringField(default = '') lng = StringField(default = '') meta = { 'collection': 'ChargeStation', 'db_alias': 'spider', 'unique_together': {'lat', 'lng'} } def get_page(regCode,pageNum): baiduUrl = 'http://api.map.baidu.com/place/v2/search?query=充电桩&page_num=%s®ion=%s&output=json&ak=3RVEkGkdaiSnVHn0dT5EGZu1RngcFlyP' % (pageNum,regCode) return json.loads(requests.get(baiduUrl,timeout=15).text) def upsert_gov(result): for info in result['results']: if 'location' not in info: continue newSch = ChargeStation(province = info.get('province',''),city = info.get('city',''),name = info.get('name',''),address = info.get('address',''),telephone=info.get('telephone',''),lat = str(info['location']['lat']),lng = str(info['location']['lng'])) try: newSch.save() except Exception,e: continue pageNum = 0 for reg in regCodes: print reg getCount = 0 pageNum = 0 result = get_page(reg,pageNum) getCount += len(result['results']) upsert_gov(result) while getCount < result['total']: pageNum += 1 result = get_page(reg,pageNum) upsert_gov(result) getCount += len(result['results']) print reg,getCount print 'ok'