manyi/script/transaction/parse_dealer_ids.py

# -*- coding: utf-8 -*-
# !/usr/bin/env python

import os

import pandas as pd
import simplejson as json

total = 0
split = {}

file_type_list = ['csv']

def get_file_list(folder):
    filelist = []  #存储要copy的文件全名
    for dirpath,dirnames,filenames in os.walk(folder):
        for file in filenames:
            file_type = file.split('.')[-1]
            if(file_type in file_type_list):
                file_fullname = os.path.join(dirpath, file) #文件全名
                filelist.append(file_fullname)
    return filelist

filelist = get_file_list('/var/www/UserServer/script/transaction')

dealerids = set()

for file in filelist:
    ttt = pd.read_csv(file)

    for i in ttt.index.values:
        id_dicts = json.loads(str(ttt.ix[i, 3]))
        for id_dict in id_dicts:
            dealerids.add(id_dict["$oid"])

for dealerid in dealerids:
    print dealerid