日志服务

阿里云日志服务

日志服务（Log Service，简称 LOG）是针对日志类数据的一站式服务，在阿里巴巴集团经历大量大数据场景锤炼而成。您无需开发就能快捷完成日志数据采集、消费、投递以及查询分析等功能，提升运维、运营效率，建立 DT 时代海量日志处理能力。

开发中日志是我们保证系统异常时，快速修复的参考依据，所以日志的相关记录，排查有时决定你项目的SLA。使用阿里云的日志服务托管日志

使用Logback Appender

** 在阿里云控制台创建project 和logstore 。记得开启索引**

依赖

<dependency>
            <groupId>com.aliyun.openservices</groupId>
            <artifactId>aliyun-log</artifactId>
            <version>0.6.31</version>
        </dependency>
        <dependency>
            <groupId>com.aliyun.openservices</groupId>
            <artifactId>aliyun-log-producer</artifactId>
            <version>0.2.0</version>
        </dependency>
      <dependency>
            <groupId>joda-time</groupId>
            <artifactId>joda-time</artifactId>
            <version>2.9.9</version>
        </dependency>

spring-logback.xml

 <appender name="aliyun" class="com.alibaba.ib.device.start.LoghubAppender">
        <!--必选项-->
        <!-- 账号及网络配置 -->
        <endpoint></endpoint>
        <accessKeyId></accessKeyId>
        <accessKeySecret></accessKeySecret>

        <!-- sls 项目配置 -->
        <project></project>
        <logStore></logStore>
        <!--必选项 (end)-->

        <!-- 可选项 -->
        <topic></topic>

        <!-- 可选项 详见 '参数说明'-->
        <totalSizeInBytes>104857600</totalSizeInBytes>
        <maxBlockMs>60000</maxBlockMs>
        <ioThreadCount>8</ioThreadCount>
        <batchSizeThresholdInBytes>524288</batchSizeThresholdInBytes>
        <batchCountThreshold>4096</batchCountThreshold>
        <lingerMs>2000</lingerMs>
        <retries>10</retries>
        <baseRetryBackoffMs>100</baseRetryBackoffMs>
        <maxRetryBackoffMs>100</maxRetryBackoffMs>

        <!-- 可选项 设置时间格式 -->
        <timeFormat>yyyy-MM-dd'T'HH:mmZ</timeFormat>
        <timeZone>GMT+08</timeZone>

        <encoder>
            <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg</pattern>
        </encoder>
        <mdcFields>THREAD_ID,MDC_KEY</mdcFields>
    </appender>

    <root level="INFO">
        <level value="INFO"/>
        <appender-ref ref="aliyun"/>
    </root>

public class LoghubAppender<E> extends UnsynchronizedAppenderBase<E> {

    private String project;

    private String endpoint;

    private String accessKeyId;

    private String accessKeySecret;

    private String userAgent = "logback";

    protected Encoder<E> encoder;

    protected ProducerConfig producerConfig = new ProducerConfig(new ProjectConfigs());
    protected ProjectConfig projectConfig;

    protected Producer producer;

    protected String logStore; //
    protected String topic = ""; //
    protected String source = ""; //

    protected String timeZone = "UTC";
    protected String timeFormat = "yyyy-MM-dd'T'HH:mmZ";
    protected DateTimeFormatter formatter;

    protected java.time.format.DateTimeFormatter formatter1;
    private String mdcFields;

    @Override
    public void start() {
        try {
            doStart();
        } catch (Exception e) {
            addError("Failed to start LoghubAppender.", e);
        }
    }

    private void doStart() {
        try {
            formatter = DateTimeFormat.forPattern(timeFormat).withZone(DateTimeZone.forID(timeZone));
        }catch (Exception e){
            formatter1 = java.time.format.DateTimeFormatter.ofPattern(timeFormat).withZone(ZoneId.of(timeZone));
        }
        producer = createProducer();
        super.start();
    }

    public Producer createProducer() {
        projectConfig = buildProjectConfig();
        producerConfig.getProjectConfigs().put(projectConfig);
        return new LogProducer(producerConfig);
    }

    private ProjectConfig buildProjectConfig() {
        return new ProjectConfig(project, endpoint, accessKeyId, accessKeySecret, null, userAgent);
    }

    @Override
    public void stop() {
        try {
            doStop();
        } catch (Exception e) {
            addError("Failed to stop LoghubAppender.", e);
        }
    }

    private void doStop() throws InterruptedException, ProducerException {
        if (!isStarted()) {
            return;
        }

        super.stop();
        producer.close();
    }

    @Override
    public void append(E eventObject) {
        try {
            appendEvent(eventObject);
        } catch (Exception e) {
            addError("Failed to append event.", e);
        }
    }

    private void appendEvent(E eventObject) {
        //init Event Object
        if (!(eventObject instanceof LoggingEvent)) {
            return;
        }
        LoggingEvent event = (LoggingEvent) eventObject;

        List<LogItem> logItems = new ArrayList<LogItem>();
        LogItem item = new LogItem();
        logItems.add(item);
        item.SetTime((int) (event.getTimeStamp() / 1000));

        if(formatter!=null){
            DateTime dateTime = new DateTime(event.getTimeStamp());
            item.PushBack("time", dateTime.toString(formatter));
        }else {
            Instant instant = Instant.ofEpochMilli(event.getTimeStamp());
            item.PushBack("time", formatter1.format(instant));
        }

        item.PushBack("level", event.getLevel().toString());
        item.PushBack("thread", event.getThreadName());

        StackTraceElement[] caller = event.getCallerData();
        if (caller != null && caller.length > 0) {
            item.PushBack("location", caller[0].toString());
        }

        String message = event.getFormattedMessage();
        item.PushBack("message", message);

        IThrowableProxy iThrowableProxy = event.getThrowableProxy();
        if (iThrowableProxy != null) {
            String throwable = getExceptionInfo(iThrowableProxy);
            throwable += fullDump(event.getThrowableProxy().getStackTraceElementProxyArray());
            item.PushBack("throwable", throwable);
        }

        if (this.encoder != null) {
            item.PushBack("log", new String(this.encoder.encode(eventObject)));
        }

        Optional.ofNullable(mdcFields).ifPresent(
                f->event.getMDCPropertyMap().entrySet().stream()
                        .filter(v->Arrays.stream(f.split(",")).anyMatch(i->i.equals(v.getKey())))
                        .forEach(map-> item.PushBack(map.getKey(),map.getValue()))
        );
        try {
            producer.send(projectConfig.getProject(), logStore, topic, source, logItems, new LoghubAppenderCallback<E>(this,
                    projectConfig.getProject(), logStore, topic, source, logItems));
        } catch (Exception e) {
            this.addError(
                    "Failed to send log, project=" + project
                            + ", logStore=" + logStore
                            + ", topic=" + topic
                            + ", source=" + source
                            + ", logItem=" + logItems, e);
        }
    }

    public String getTimeFormat() {
        return timeFormat;
    }

    public void setTimeFormat(String timeFormat) {
        this.timeFormat = timeFormat;
    }

    private String getExceptionInfo(IThrowableProxy iThrowableProxy) {
        String s = iThrowableProxy.getClassName();
        String message = iThrowableProxy.getMessage();
        return (message != null) ? (s + ": " + message) : s;
    }

    private String fullDump(StackTraceElementProxy[] stackTraceElementProxyArray) {
        StringBuilder builder = new StringBuilder();
        for (StackTraceElementProxy step : stackTraceElementProxyArray) {
            builder.append(CoreConstants.LINE_SEPARATOR);
            String string = step.toString();
            builder.append(CoreConstants.TAB).append(string);
            ThrowableProxyUtil.subjoinPackagingData(builder, step);
        }
        return builder.toString();
    }

    public String getLogStore() {
        return logStore;
    }

    public void setLogStore(String logStore) {
        this.logStore = logStore;
    }

    public String getTopic() {
        return topic;
    }

    public void setTopic(String topic) {
        this.topic = topic;
    }

    public String getSource() {
        return source;
    }

    public void setSource(String source) {
        this.source = source;
    }

    public String getTimeZone() {
        return timeZone;
    }

    public void setTimeZone(String timeZone) {
        this.timeZone = timeZone;
    }

    // **** ==- ProjectConfig -== **********************
    public String getProject() {
        return project;
    }

    public void setProject(String project) {
        this.project = project;
    }

    public String getEndpoint() {
        return endpoint;
    }

    public void setEndpoint(String endpoint) {
        this.endpoint = endpoint;
    }

    public String getAccessKeyId() {
        return accessKeyId;
    }

    public void setAccessKeyId(String accessKeyId) {
        this.accessKeyId = accessKeyId;
    }

    public String getAccessKeySecret() {
        return accessKeySecret;
    }

    public void setAccessKeySecret(String accessKeySecret) {
        this.accessKeySecret = accessKeySecret;
    }

    public String getUserAgent() {
        return userAgent;
    }

    public void setUserAgent(String userAgent) {
        this.userAgent = userAgent;
    }

    public int getTotalSizeInBytes() {
        return producerConfig.getTotalSizeInBytes();
    }

    public void setTotalSizeInBytes(int totalSizeInBytes) {
        producerConfig.setTotalSizeInBytes(totalSizeInBytes);
    }

    public long getMaxBlockMs() {
        return producerConfig.getMaxBlockMs();
    }

    public void setMaxBlockMs(long maxBlockMs) {
        producerConfig.setMaxBlockMs(maxBlockMs);
    }

    public int getIoThreadCount() {
        return producerConfig.getIoThreadCount();
    }

    public void setIoThreadCount(int ioThreadCount) {
        producerConfig.setIoThreadCount(ioThreadCount);
    }

    public int getBatchSizeThresholdInBytes() {
        return producerConfig.getBatchSizeThresholdInBytes();
    }

    public void setBatchSizeThresholdInBytes(int batchSizeThresholdInBytes) {
        producerConfig.setBatchSizeThresholdInBytes(batchSizeThresholdInBytes);
    }

    public int getBatchCountThreshold() {
        return producerConfig.getBatchCountThreshold();
    }

    public void setBatchCountThreshold(int batchCountThreshold) {
        producerConfig.setBatchCountThreshold(batchCountThreshold);
    }

    public int getLingerMs() {
        return producerConfig.getLingerMs();
    }

    public void setLingerMs(int lingerMs) {
        producerConfig.setLingerMs(lingerMs);
    }

    public int getRetries() {
        return producerConfig.getRetries();
    }

    public void setRetries(int retries) {
        producerConfig.setRetries(retries);
    }

    public int getMaxReservedAttempts() {
        return producerConfig.getMaxReservedAttempts();
    }

    public void setMaxReservedAttempts(int maxReservedAttempts) {
        producerConfig.setMaxReservedAttempts(maxReservedAttempts);
    }

    public long getBaseRetryBackoffMs() {
        return producerConfig.getBaseRetryBackoffMs();
    }

    public void setBaseRetryBackoffMs(long baseRetryBackoffMs) {
        producerConfig.setBaseRetryBackoffMs(baseRetryBackoffMs);
    }

    public long getMaxRetryBackoffMs() {
        return producerConfig.getMaxRetryBackoffMs();
    }

    public void setMaxRetryBackoffMs(long maxRetryBackoffMs) {
        producerConfig.setMaxRetryBackoffMs(maxRetryBackoffMs);
    }

    public Encoder<E> getEncoder() {
        return encoder;
    }

    public void setEncoder(Encoder<E> encoder) {
        this.encoder = encoder;
    }

    public void setMdcFields(String mdcFields) {
        this.mdcFields = mdcFields;
    }
}

public class LoghubAppenderCallback<E> implements Callback {

    protected LoghubAppender<E> loghubAppender;

    protected String project;

    protected String logstore;

    protected String topic;

    protected String source;

    protected List<LogItem> logItems;

    public LoghubAppenderCallback(LoghubAppender<E> loghubAppender, String project, String logstore, String topic,
                                  String source, List<LogItem> logItems) {
        super();
        this.loghubAppender = loghubAppender;
        this.project = project;
        this.logstore = logstore;
        this.topic = topic;
        this.source = source;
        this.logItems = logItems;
    }

    @Override
    public void onCompletion(Result result) {
        if (!result.isSuccessful()) {
            loghubAppender.addError(
                    "Failed to send log, project=" + project
                            + ", logStore=" + logstore
                            + ", topic=" + topic
                            + ", source=" + source
                            + ", logItem=" + logItems
                            + ", errorCode=" + result.getErrorCode()
                            + ", errorMessage=" + result.getErrorMessage());
        }
    }
}

Test

public class TestAppender {

    private static final Logger LOGGER = LoggerFactory.getLogger(TestAppender.class);

    @Test
    public void testLogCommonMessage() {
        LOGGER.warn("This is a sls message logged by logback.");
    }
}

执行结果

image.png

日志服务操作实践

上下文查询、快速查询、快速分析
支持多种检索查询语法：语法