Graylog 合并多行 Java Error 日志
2018-12-03 本文已影响11人
awker
1、线上运行的 java 服务输出的 error 日志格式为（以其中2条日志记录作示例）
[tx-web-fms]2018-12-03 09:52:00,007 CST ERROR com.alibaba.druid.filter.logging.Slf4jLogFilter 149 statementLogError - {conn-10205, stmt-26304} execute error. SELECT 1 com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: Communications link failure
The last packet successfully received from the server was 1,199,997 milliseconds ago. The last packet sent successfully to the server was 1,199,998 milliseconds ago.
at sun.reflect.GeneratedConstructorAccessor96.newInstance(Unknown Source) ~[?:?]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_181]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_181]
at com.mysql.jdbc.Util.handleNewInstance(Util.java:377) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1036) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:3661) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2417) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2582) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2526) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2484) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.StatementImpl.executeQuery(StatementImpl.java:1446) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.alibaba.druid.filter.FilterChainImpl.statement_executeQuery(FilterChainImpl.java:2363) ~[druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterAdapter.statement_executeQuery(FilterAdapter.java:2481) ~[druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterEventAdapter.statement_executeQuery(FilterEventAdapter.java:302) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.statement_executeQuery(FilterChainImpl.java:2360) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterAdapter.statement_executeQuery(FilterAdapter.java:2481) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterEventAdapter.statement_executeQuery(FilterEventAdapter.java:302) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.statement_executeQuery(FilterChainImpl.java:2360) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.proxy.jdbc.StatementProxyImpl.executeQuery(StatementProxyImpl.java:211) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.ValidConnectionCheckerAdapter.isValidConnection(ValidConnectionCheckerAdapter.java:44) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidAbstractDataSource.testConnectionInternal(DruidAbstractDataSource.java:1298) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1054) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4544) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.logging.LogFilter.dataSource_getConnection(LogFilter.java:831) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4540) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.stat.StatFilter.dataSource_getConnection(StatFilter.java:662) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4540) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:992) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:984) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:103) [druid-1.0.25.jar:1.0.25]
at org.springframework.jdbc.datasource.DataSourceTransactionManager.doBegin(DataSourceTransactionManager.java:204) [spring-jdbc-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:427) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:276) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) [spring-aop-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) [spring-aop-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at com.sun.proxy.$Proxy86.synRefundBondStatus(Unknown Source) [?:?]
at com.tx.web.fms.service.elastic.SynRefundBondStatusJob.execute(SynRefundBondStatusJob.java:23) [classes/:?]
at com.dangdang.ddframe.job.executor.type.SimpleJobExecutor.process(SimpleJobExecutor.java:41) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.process(AbstractElasticJobExecutor.java:206) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.process(AbstractElasticJobExecutor.java:171) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.execute(AbstractElasticJobExecutor.java:150) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.execute(AbstractElasticJobExecutor.java:122) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.lite.internal.schedule.LiteJob.execute(LiteJob.java:26) [elastic-job-lite-core-2.1.5.jar:?]
at org.quartz.core.JobRunShell.run(JobRunShell.java:202) [quartz-2.2.1.jar:?]
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:573) [quartz-2.2.1.jar:?]
Caused by: java.net.SocketException: Connection reset
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:115) ~[?:1.8.0_181]
at java.net.SocketOutputStream.write(SocketOutputStream.java:155) ~[?:1.8.0_181]
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:3643) ~[mysql-connector-java-5.1.34.jar:5.1.34]
... 41 more
[tx-web-fms]2018-12-03 16:32:00,008 CST ERROR com.alibaba.druid.filter.logging.Slf4jLogFilter 149 statementLogError - {conn-10203, stmt-26390} execute error. SELECT 1 com.mysql.jdbc.exceptions.jdbc4.CommunicationsException: Communications link failure
The last packet successfully received from the server was 1,199,996 milliseconds ago. The last packet sent successfully to the server was 1,199,996 milliseconds ago.
at sun.reflect.GeneratedConstructorAccessor94.newInstance(Unknown Source) ~[?:?]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_181]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_181]
at com.mysql.jdbc.Util.handleNewInstance(Util.java:377) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.SQLError.createCommunicationsException(SQLError.java:1036) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:3661) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2417) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2582) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2526) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2484) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.mysql.jdbc.StatementImpl.executeQuery(StatementImpl.java:1446) ~[mysql-connector-java-5.1.34.jar:5.1.34]
at com.alibaba.druid.filter.FilterChainImpl.statement_executeQuery(FilterChainImpl.java:2363) ~[druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterAdapter.statement_executeQuery(FilterAdapter.java:2481) ~[druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterEventAdapter.statement_executeQuery(FilterEventAdapter.java:302) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.statement_executeQuery(FilterChainImpl.java:2360) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterAdapter.statement_executeQuery(FilterAdapter.java:2481) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterEventAdapter.statement_executeQuery(FilterEventAdapter.java:302) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.statement_executeQuery(FilterChainImpl.java:2360) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.proxy.jdbc.StatementProxyImpl.executeQuery(StatementProxyImpl.java:211) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.ValidConnectionCheckerAdapter.isValidConnection(ValidConnectionCheckerAdapter.java:44) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidAbstractDataSource.testConnectionInternal(DruidAbstractDataSource.java:1298) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:1054) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4544) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.logging.LogFilter.dataSource_getConnection(LogFilter.java:831) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4540) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.stat.StatFilter.dataSource_getConnection(StatFilter.java:662) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.filter.FilterChainImpl.dataSource_connect(FilterChainImpl.java:4540) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:992) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:984) [druid-1.0.25.jar:1.0.25]
at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:103) [druid-1.0.25.jar:1.0.25]
at org.springframework.jdbc.datasource.DataSourceTransactionManager.doBegin(DataSourceTransactionManager.java:204) [spring-jdbc-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.support.AbstractPlatformTransactionManager.getTransaction(AbstractPlatformTransactionManager.java:373) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.interceptor.TransactionAspectSupport.createTransactionIfNecessary(TransactionAspectSupport.java:427) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:276) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.transaction.interceptor.TransactionInterceptor.invoke(TransactionInterceptor.java:96) [spring-tx-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) [spring-aop-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:207) [spring-aop-4.2.0.RELEASE.jar:4.2.0.RELEASE]
at com.sun.proxy.$Proxy37.synAcWithdrawStatus(Unknown Source) [?:?]
at com.tx.web.fms.service.elastic.SynAcWithdrawStatusJob.execute(SynAcWithdrawStatusJob.java:23) [classes/:?]
at com.dangdang.ddframe.job.executor.type.SimpleJobExecutor.process(SimpleJobExecutor.java:41) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.process(AbstractElasticJobExecutor.java:206) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.process(AbstractElasticJobExecutor.java:171) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.execute(AbstractElasticJobExecutor.java:150) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.executor.AbstractElasticJobExecutor.execute(AbstractElasticJobExecutor.java:122) [elastic-job-common-core-2.1.5.jar:?]
at com.dangdang.ddframe.job.lite.internal.schedule.LiteJob.execute(LiteJob.java:26) [elastic-job-lite-core-2.1.5.jar:?]
at org.quartz.core.JobRunShell.run(JobRunShell.java:202) [quartz-2.2.1.jar:?]
at org.quartz.simpl.SimpleThreadPool$WorkerThread.run(SimpleThreadPool.java:573) [quartz-2.2.1.jar:?]
Caused by: java.net.SocketException: Connection reset
at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:115) ~[?:1.8.0_181]
at java.net.SocketOutputStream.write(SocketOutputStream.java:155) ~[?:1.8.0_181]
at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) ~[?:1.8.0_181]
at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) ~[?:1.8.0_181]
at com.mysql.jdbc.MysqlIO.send(MysqlIO.java:3643) ~[mysql-connector-java-5.1.34.jar:5.1.34]
... 41 more
2、graylog 配置 filebeat 的多行匹配规则
查看生成的规则
3、测试是否成功
可以看到已经成功合并了多行 java error 日志
4、filebeat 参数说明
multiline:
match: after
negate: true
pattern: ^\[tx
multiline.pattern: 设置行的匹配字段
multiline.negate: 设置符合上面匹配条件的的行，是否应该被合并成一条日志。默认是 false，匹配 pattern 的行合并到上一行；true，不匹配 pattern 的行合并到上一行
multiline.match: 设置符合 pattern 的行应该被合并到之前不符合 pattern 的行，还是之后不符合 pattern 的行
