非交织mp4与short_seek_threshold

前段时间排查一个播放卡顿的问题，本地播放又是正常的。通过分析mp4文件，发现该文件与一般mp4文件有所不同。

通常，mp4文件的mdat区，视频和音频是交织存放的。这样在读的时候就能很方便的把当前要用的的声音和视频一起读出来。

然而，mp4的规范并没有限定一定要这样存放，它们可以单独分开位于文件的不同位置，只要在索引中能找到对应的包即可。于是就有非交织的情况发生。

两种不同的存储方式

生成非交织的文件比交织的更复杂，需要写完一条track后才能写另一条track，可能需要事先知道track的总大小。这种情况一般出现在视频编辑的场景中。

FFmpeg在解non-interleaved文件时，需要不断的在两个track中来回seek，对应的网络请求的次数也会大大增加，导致加载时间变长。

在测试过程中，发现某些非交织文件使用自定义的protocal并没有出现多次seek的情况。这让我很奇怪，因为自定义的protocal也是调用http URLContext。经过一番调试，终于找到了猫腻。

int64_t avio_seek(AVIOContext *s, int64_t offset, int whence)
{
    int64_t offset1;
    int64_t pos;
    int force = whence & AVSEEK_FORCE;
    int buffer_size;
    int short_seek;
    whence &= ~AVSEEK_FORCE;

    if(!s)
        return AVERROR(EINVAL);

    buffer_size = s->buf_end - s->buffer;
    // pos is the absolute position that the beginning of s->buffer corresponds to in the file
    pos = s->pos - (s->write_flag ? 0 : buffer_size);

    if (whence != SEEK_CUR && whence != SEEK_SET)
        return AVERROR(EINVAL);

    if (whence == SEEK_CUR) {
        offset1 = pos + (s->buf_ptr - s->buffer);
        if (offset == 0)
            return offset1;
        offset += offset1;
    }
    if (offset < 0)
        return AVERROR(EINVAL);

    if (s->short_seek_get) {
        short_seek = s->short_seek_get(s->opaque);
        /* fallback to default short seek */
        if (short_seek <= 0)
            short_seek = s->short_seek_threshold;
    } else
        short_seek = s->short_seek_threshold;

    offset1 = offset - pos; // "offset1" is the relative offset from the beginning of s->buffer
    if (!s->must_flush && (!s->direct || !s->seek) &&
        offset1 >= 0 && offset1 <= buffer_size - s->write_flag) {
        /* can do the seek inside the buffer */
        s->buf_ptr = s->buffer + offset1;
    } else if ((!(s->seekable & AVIO_SEEKABLE_NORMAL) ||
               offset1 <= buffer_size + short_seek) &&
               !s->write_flag && offset1 >= 0 &&
               (!s->direct || !s->seek) &&
              (whence != SEEK_END || force)) {
        while(s->pos < offset && !s->eof_reached)
            fill_buffer(s);
        if (s->eof_reached)
            return AVERROR_EOF;
        s->buf_ptr = s->buf_end - (s->pos - offset);
    } else if(!s->write_flag && offset1 < 0 && -offset1 < buffer_size>>1 && s->seek && offset > 0) {
        int64_t res;

        pos -= FFMIN(buffer_size>>1, pos);
        if ((res = s->seek(s->opaque, pos, SEEK_SET)) < 0)
            return res;
        s->buf_end =
        s->buf_ptr = s->buffer;
        s->pos = pos;
        s->eof_reached = 0;
        fill_buffer(s);
        return avio_seek(s, offset, SEEK_SET | force);
    } else {
        int64_t res;
        if (s->write_flag) {
            flush_buffer(s);
            s->must_flush = 1;
        }
        if (!s->seek)
            return AVERROR(EPIPE);
        if ((res = s->seek(s->opaque, offset, SEEK_SET)) < 0)
            return res;
        s->seek_count ++;
        if (!s->write_flag)
            s->buf_end = s->buffer;
        s->buf_ptr = s->buffer;
        s->pos = offset;
    }
    s->eof_reached = 0;
    return offset;
}

在avio_seek中有这么一行，offset1 <= buffer_size + short_seek。意思是当demux需要seek的位置超过buffer_size，但小于buffer_size + short_seek时，不重新seek而是继续往下读。显然，这是专门优化多次seek的问题。

默认情况下，short_seek的值从s->short_seek_get(s->opaque)获取，它最终是获取当前tcp滑动窗口的大小，调试的时候发现这个值最多也就1M左右。

static int tcp_get_window_size(URLContext *h)
{
    TCPContext *s = h->priv_data;
    int avail;
    int avail_len = sizeof(avail);

#if HAVE_WINSOCK2_H
    /* SO_RCVBUF with winsock only reports the actual TCP window size when
    auto-tuning has been disabled via setting SO_RCVBUF */
    if (s->recv_buffer_size < 0) {
        return AVERROR(ENOSYS);
    }
#endif

    if (getsockopt(s->fd, SOL_SOCKET, SO_RCVBUF, &avail, &avail_len)) {
        return ff_neterrno();
    }
    return avail;
}

自定义的protocol没有实现short_seek_get，所以它用的是s->short_seek_threshold默认值。这个值初始值很小，只有4k。但当解析完header后，FFmpeg会从新计算这个值。

void ff_configure_buffers_for_index(AVFormatContext *s, int64_t time_tolerance)
{
    int ist1, ist2;
    int64_t pos_delta = 0;
    int64_t skip = 0;
    //We could use URLProtocol flags here but as many user applications do not use URLProtocols this would be unreliable
    const char *proto = avio_find_protocol_name(s->filename);

    if (!proto) {
        av_log(s, AV_LOG_INFO,
               "Protocol name not provided, cannot determine if input is local or "
               "a network protocol, buffers and access patterns cannot be configured "
               "optimally without knowing the protocol\n");
    }

    if (proto && !(strcmp(proto, "file") && strcmp(proto, "pipe") && strcmp(proto, "cache")))
        return;
    // 下面这个大循环用来计算相邻dts不同track的最大间距（pos_delta），以及最大entry的size（skip）
    for (ist1 = 0; ist1 < s->nb_streams; ist1++) {
        AVStream *st1 = s->streams[ist1];
        for (ist2 = 0; ist2 < s->nb_streams; ist2++) {
            AVStream *st2 = s->streams[ist2];
            int i1, i2;

            if (ist1 == ist2)
                continue;

            for (i1 = i2 = 0; i1 < st1->nb_index_entries; i1++) {
                AVIndexEntry *e1 = &st1->index_entries[i1];
                int64_t e1_pts = av_rescale_q(e1->timestamp, st1->time_base, AV_TIME_BASE_Q);

                skip = FFMAX(skip, e1->size);
                for (; i2 < st2->nb_index_entries; i2++) {
                    AVIndexEntry *e2 = &st2->index_entries[i2];
                    int64_t e2_pts = av_rescale_q(e2->timestamp, st2->time_base, AV_TIME_BASE_Q);
                    if (e2_pts - e1_pts < time_tolerance)
                        continue;
                    pos_delta = FFMAX(pos_delta, e1->pos - e2->pos);
                    break;
                }
            }
        }
    }

    pos_delta *= 2;
    /* XXX This could be adjusted depending on protocol*/
    if (s->pb->buffer_size < pos_delta && pos_delta < (1<<24)) {
        av_log(s, AV_LOG_VERBOSE, "Reconfiguring buffers to size %"PRId64"\n", pos_delta);
        ffio_set_buf_size(s->pb, pos_delta);
        s->pb->short_seek_threshold = FFMAX(s->pb->short_seek_threshold, pos_delta/2);
    }

    if (skip < (1<<23)) {
        s->pb->short_seek_threshold = FFMAX(s->pb->short_seek_threshold, skip);
    }
}

后面有一个判断，最大间距不超过8M，avio_seek的seek优化才能生效。这就解释了，为什么我们自定义的protocol只对部分非交织的文件mp4有效。

可以把这个阈值设大，但随之而来的问题是加载时间变长。最优的解法是用双缓冲来解析这种非交织的文件，具体原理将在下篇文件中介绍，敬请期待。