manyi/script/eventer/parse_event_log.py

# coding=utf-8
import os, sys
import subprocess
from collections import Counter
import datetime


PROJECT_ROOT = os.path.join(os.path.abspath(os.path.split(os.path.realpath(__file__))[0] + "/.."), '..')
sys.path.insert(0, PROJECT_ROOT)
os.environ["DJANGO_SETTINGS_MODULE"] = "configs.production"


from script.base import init_env


init_env(False)


from apps.web.device.models import Device, DeviceDict, EventTimes


def carry_out_command(logs, st, et):  # type: (str, str, str) -> list
    """
    执行筛选日志的命令
    st et 为时间格式 只能到小时 YYYY-MM-DD HH

    """
    # 该指令的作用是 筛选出所有的时间段内的日志 然后删选出100事件 最后筛选出设备的编号 两个管道处理
    cmd = "sed -n '/%s*/,/%s*/p' %s | grep -oP 'server/\d{15}/100' | cut -c8-22" % (st, et, logs)
    s = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True)

    devNos = list()
    for line in s.stdout.readlines():
        devNos.append(line)
    return devNos


def dump_data(devNo, times, date):
    dev = Device.get_dev(devNo) # type: DeviceDict
    if not dev:
        server = ""
        devTypeCode = ""
        driverCode = ""
        driverVersion = ""
    else:
        server = dev.server
        devTypeCode = dev.devTypeCode
        driverCode = dev.driverCode
        driverVersion = dev.driverVersion

    # EventTimes(devNo=devNo, driverCode=driverCode, driverVersion=driverVersion, devTypeCode=devTypeCode, server=server, times=times, date=date).save()

    return "times = {}, devNo = {}, devType = {}, driver = {}, server = {}, date = {}".format(times, devNo, devTypeCode, driverCode, server, date)


def main():
    LOG_FILES = [
        "/var/log/washpayer/eventer/1883/listener/server.log",
        "/var/log/washpayer/eventer/1884/listener/server.log",
        "/var/log/washpayer/eventer/1885/listener/server.log",
        "/var/log/washpayer/eventer/1886/listener/server.log",
        "/var/log/washpayer/eventer/1887/listener/server.log",

        "/var/log/washpayer/eventer/1883/listener/server.log.?",
        "/var/log/washpayer/eventer/1884/listener/server.log.?",
        "/var/log/washpayer/eventer/1885/listener/server.log.?",
        "/var/log/washpayer/eventer/1886/listener/server.log.?",
        "/var/log/washpayer/eventer/1887/listener/server.log.?",

        "/var/log/washpayer/eventer/1883/listener/server.log.1?",
        "/var/log/washpayer/eventer/1884/listener/server.log.1?",
        "/var/log/washpayer/eventer/1885/listener/server.log.1?",
        "/var/log/washpayer/eventer/1886/listener/server.log.1?",
        "/var/log/washpayer/eventer/1887/listener/server.log.1?",

        "/var/log/washpayer/eventer/1883/listener/server.log.2?",
        "/var/log/washpayer/eventer/1884/listener/server.log.2?",
        "/var/log/washpayer/eventer/1885/listener/server.log.2?",
        "/var/log/washpayer/eventer/1886/listener/server.log.2?",
        "/var/log/washpayer/eventer/1887/listener/server.log.2?",
    ]
    logs = " ".join(LOG_FILES)

    # 时间是永远收集上一个小时的 1点执行，收集12点-1点的 2点20执行 收集1点-2点的
    now = datetime.datetime.now()
    st = now - datetime.timedelta(hours=1)
    et = now - datetime.timedelta(hours=0)

    devNos = carry_out_command(logs, st.strftime("%Y-%m-%d %H"), et.strftime("%Y-%m-%d %H"))
    result = Counter(devNos).most_common(100)

    for _devNo, _times in result:
        dump_data(_devNo.strip(), _times, st.strftime("%m%d%H"))


if __name__ == '__main__':
    main()