基于python实现将日志收集

实现的功能：

• 将远端指定服务器的日志拷贝到当前服务器行
• 将拷贝过来的日志进行压缩便于拷贝
  目录结构：
  [ansible@java2 log-gather]$ tree .
  .
  ├── action
  │ ├── init.py
  │ ├── init.pyc
  │ ├── rsync_log.py
  │ └── rsync_log.pyc
  ├── log-gather.py
  └── requirement.txt

1 directory, 6 files

一、所需的模块

[ansible@java2 log-gather]$ cat requirement.txt 
python3
docopt
sh

二、入口文件

[ansible@java2 log-gather]$ cat log-gather.py 
#! /home/ansible/.venv/bin/python
# - * - coding: utf-8 - * -

"""LOG-GATEHER-CLI CLI.
Usage:[]
    log-gather.py copylog --host=<kn> --program=<kn>
    log-gather.py packlog-internal --host=<kn> --program=<kn>
    log-gather.py cleanlog  --host=<kn> --program=<kn>
    log-gather.py -h | --help
    log-gather.py --version

Options:
    copylog                  cp remote log file to java2 tmp dir.
    packlog-internal         compress log file to tar.gz
    cleanlog                 clean old log file in local path.
    --host=<kn>              input host name,include java1,java2,java3,and java4.
    --program=<kn>           input program name ,for example spring, play and so on.
    -h, --help               display this help and exit.
    --version                output version information and exit.

"""

from docopt import docopt
from action import rsync_log


def action_route(doc_args):
    if doc_args.get("copylog"): 
        rsync_log.CpLogTask(doc_args.get("--host"), doc_args.get("--program"))
    elif doc_args.get("packlog-internal"):
        rsync_log.PackLog(doc_args.get("--host"), doc_args.get("--program"))
    elif doc_args.get("cleanlog"):
        rsync_log.CleanOld_log(doc_args.get("--host"), doc_args.get("--program"))         
    else:
        print("An unreasonable parameters")
    

if __name__ == '__main__':
    args = docopt(__doc__, version='LOG-GATEHER-CLI 1.0')
    action_route(args)

三、执行文件

 cat  action/rsync_log.py
# - * - coding: utf-8 - * -

# import conf
import datetime
import sys
import os
import sh
from sh import cd

TEMP_DIR = '/tmp'
Date = datetime.datetime.now().strftime('%Y-%m-%d_%H_%M_%S')
Logdata = datetime.datetime.now().strftime('%Y-%m-%d')
scp_cmd = sh.Command("/usr/bin/scp")
remote_log_path = ''
local_path = ''
logname = ''
logname1 = ''
logname2 = ''


def Check_args(hostname, program):
    '''检测参数格式是否正确'''

    global remote_log_path, logname, logname1, logname2
    if hostname != 'java1' and hostname != 'java2' and hostname != 'java3' and hostname != 'java4':
        print('''Please input right hostname,Include 'java1','java2','java3' or 'java4' in command ''')
        sys.exit(1)

    if (hostname == 'java1' or hostname == 'java2') and program != 'play':
        print("please check the relationship of hostname with program")
        sys.exit(1)
    elif (hostname == 'java3' or hostname == 'java4') and program != 'spring':
        print("please check the relationship of hostname with program")    
        sys.exit(1)

    if program == 'spring':
        remote_log_path = "/data/tomcat/logs"
        logname1 = "service.log"
        logname2 = "query.log"
    elif program == 'play':
        remote_log_path = "/data/tomcat/logs"
        logname = 'catalina_%s.%s.out' % (hostname, Logdata)
    else:
        print('''Please input right program name,For example 'play' or 'spring' in command ''')
        sys.exit(1)


def Spring_program(hostname, program):
    '''spring 项目日志文件复制'''
    if program == 'spring':
        output = scp_cmd("-P22", "%s:%s/%s" % (hostname, remote_log_path, logname1), "%s/%s_%s" % (local_path, Date, logname1))
        if output.exit_code:
            print("cp %s from remote failed." % (logname1))
        else:
            print("cp %s from remote succeed." % (logname1))

        output = scp_cmd("-P22", "%s:%s/%s" % (hostname, remote_log_path, logname2), "%s/%s_%s" % (local_path, Date, logname2)) 
        if output.exit_code:
            print("cp %s from remote failed" % (logname2))
            sys.exit(1)
        else:
            print("cp %s from remote succeed. " % (logname2))      


def Play_program(hostname, program):
    '''play 项目日志文件复制'''
    if program == 'play':
        output = scp_cmd("-P22", "%s:%s/%s" % (hostname, remote_log_path, logname), "%s/%s_%s" % (local_path, Date, logname))
        if output.exit_code:
            print("cp %s from remote failed." % (logname))
        else:
            print("cp %s from remote succeed." % (logname))


def CpLogTask(hostname, program):
    '''复制日志文件到java2'''

    global local_path
    local_path = "%s/%s/%s" % (TEMP_DIR, hostname, program)
    Check_args(hostname, program)
    if os.path.exists(local_path) is not True:
        os.makedirs(local_path)
    print("mkdir local log path succeed.")   

    if program == 'spring':
        Spring_program(hostname, program)
    elif program == 'play':
        Play_program(hostname, program)

    output = os.system("/usr/bin/sudo /bin/chown  tomcat.tomcat %s/* " % (local_path) )
    if output is True:
        print("change privileges to tomcat failed.")
    else:
        print("change privileges to tomcat succeed.")


def CleanOld_log(hostname, program):
    '''清理java2上旧日志'''

    Check_args(hostname, program)
    local_path = "%s/%s/%s" % (TEMP_DIR, hostname, program)
    for filename in os.listdir(local_path):
        if filename.endswith('.log') or filename.endswith('.out'):
            path_filename = os.path.join(local_path, filename)
            os.unlink(path_filename)
            print("clean old log %s succeed." % (filename))


def PackLog(hostname, program):
    '''在java2上打包日志文件'''
    CpLogTask(hostname, program)
    local_path = "%s/%s" % (TEMP_DIR, hostname)
    cd(local_path)
    pack_name = "%s_%s.tar.gz" % (Date, program)
    tar_cmd = sh.Command("/bin/tar")
    output = tar_cmd("czf", "%s" % (pack_name), "%s" % (program))
    if output.exit_code:
        print("Compress log failed.")
    else:
        print("Compress log succeed.")