WebRtc Video Receiverwebrtc

WebRtc Video Receiver(七)-基于Kalma

2020-09-12  本文已影响0人  JeffreyLau

1）前言

2）PlayoutDelay更新

#common_types.h
struct PlayoutDelay {
  PlayoutDelay(int min_ms, int max_ms) : min_ms(min_ms), max_ms(max_ms) {}
  int min_ms;//最小播放延迟
  int max_ms;//做到播放延迟
  ....  
}    

#rtp_video_header.h
struct RTPVideoHeader {
  ....  
  PlayoutDelay playout_delay = {-1, -1};
  ....
}

# encoded_image.h
class RTC_EXPORT EncodedImage {
 public:
  ...  
  // When an application indicates non-zero values here, it is taken as an
  // indication that all future frames will be constrained with those limits
  // until the application indicates a change again.
  PlayoutDelay playout_delay_ = {-1, -1};
  ...
}

RtpFrameObject::RtpFrameObject(
   ......
    : first_seq_num_(first_seq_num),
      last_seq_num_(last_seq_num),
      last_packet_received_time_(last_packet_received_time),
      times_nacked_(times_nacked) {

  // Setting frame's playout delays to the same values
  // as of the first packet's.
  SetPlayoutDelay(rtp_video_header_.playout_delay);
  ...
}

"http://www.webrtc.org/experiments/rtp-hdrext/playout-delay"

void VideoReceiveStream2::OnCompleteFrame(
    std::unique_ptr<video_coding::EncodedFrame> frame) {
  ....
  //拿到PlayoutDelay引用    
  const PlayoutDelay& playout_delay = frame->EncodedImage().playout_delay_;
    
  if (playout_delay.min_ms >= 0) {
    frame_minimum_playout_delay_ms_ = playout_delay.min_ms;
    UpdatePlayoutDelays();
  }

  if (playout_delay.max_ms >= 0) {
    frame_maximum_playout_delay_ms_ = playout_delay.max_ms;
    UpdatePlayoutDelays();
  }
  ....
}

void VCMTiming::set_min_playout_delay(int min_playout_delay_ms) {
  rtc::CritScope cs(&crit_sect_);
  min_playout_delay_ms_ = min_playout_delay_ms;
}

void VCMTiming::set_max_playout_delay(int max_playout_delay_ms) {
  rtc::CritScope cs(&crit_sect_);
  max_playout_delay_ms_ = max_playout_delay_ms;
}

3）RenderTimeMs设置流程

int64_t FrameBuffer::FindNextFrame(int64_t now_ms) {
  ....
  //默认该函数调用到这里的时候期望渲染时间都还未赋值的    
  if (frame->RenderTime() == -1) {
      //首先调用VCMTiming获取期望渲染时间,然后将其设置到Frame中，供后续使用
      frame->SetRenderTime(timing_->RenderTimeMs(frame->Timestamp(), now_ms));
  }
  ...  
  //得出最大等待时间    
  wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms);
  ....
  //取最小时间，如果在一次调度时间（未超时范围内）的话，返回wait_ms    
  wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms_ - now_ms);
    
  wait_ms = std::max<int64_t>(wait_ms, 0);  
  return wait_ms;
}

3.1）VCMTiming模块获取期望渲染时间

int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp,
                                int64_t now_ms) const {
  rtc::CritScope cs(&crit_sect_);
  return RenderTimeMsInternal(frame_timestamp, now_ms);
}
//frame_timestamp为当前帧的时间戳以1/90k为单位,now_ms为当前Clock时间
int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp,
                                        int64_t now_ms) const {
  //如果min_playout_delay_ms_=0并且max_playout_delay_ms_=0则表示立即渲染
 // 不建议赋值0，若赋值0的话jitterDelay就失效了
  if (min_playout_delay_ms_ == 0 && max_playout_delay_ms_ == 0) {
    // Render as soon as possible.
    return 0;
  }
  //传入当前帧的时间戳，来得到一个平滑渲染时间，TimestampExtrapolator通过卡尔曼滤波负责期望接收时间的产生
  int64_t estimated_complete_time_ms =
      ts_extrapolator_->ExtrapolateLocalTime(frame_timestamp);
  if (estimated_complete_time_ms == -1) {
    estimated_complete_time_ms = now_ms;
  }

  // Make sure the actual delay stays in the range of |min_playout_delay_ms_|
  // and |max_playout_delay_ms_|.
  // 和min_playout_delay_ms_取最大值,min_playout_delay_ms_默认值-1,  
  int actual_delay = std::max(current_delay_ms_, min_playout_delay_ms_);
  //和max_playout_delay_ms_求最小值，max_playout_delay_ms_默认值-1  
  actual_delay = std::min(actual_delay, max_playout_delay_ms_);
  return estimated_complete_time_ms + actual_delay;
}

3.2）VCMTiming模块获取调度等待时间

int64_t VCMTiming::MaxWaitingTime(int64_t render_time_ms,
                                  int64_t now_ms) const {
  rtc::CritScope cs(&crit_sect_);
  const int64_t max_wait_time_ms =
      render_time_ms - now_ms - RequiredDecodeTimeMs() - render_delay_ms_;

  return max_wait_time_ms;
}

4）TimestampExtrapolator Kalman filter期望渲染时间估计

4.1）TimestampExtrapolator模块Kalman模型

  // Local time in webrtc time base.
  int64_t current_time_us = clock_->TimeInMicroseconds();
  int64_t current_time_ms = current_time_us / rtc::kNumMicrosecsPerMillisec;

  // Capture time may come from clock with an offset and drift from clock_.
  int64_t capture_ntp_time_ms = current_time_ms + delta_ntp_internal_ms_;
  // Convert NTP time, in ms, to RTP timestamp.
  const int kMsToRtpTimestamp = 90;
  uint32_t timestamp_rtp =
      kMsToRtpTimestamp * static_cast<uint32_t>(capture_ntp_time_ms);


 fps = 60fps
 samplerate = 90000    
 timestampDiff(k) = rtpTimeStamp(k) - rtpTimeStamp(0)                        (4.1.1)
 timestampDiffToMs(k) = timestampDiff(k) * 1000 / samplerate                 (4.1.2)

 t(0) = _startMs
 t(k) = timestampDiffToMs(k) + t(0)                                     (4.1.3)

 t(k) = timestampDiffToMs(k) + t(0) + error(k)                          (4.1.4)

 t(k) = (timestampDiff(k) - jitterTimestamp(k))  / sampleratePermillage(k) + t(0)   (4.1.5)

 w(k) = w(k-1) + u(k-1)                                 P(u) ~ (0,Q)    (4.1.6)      
 w_bar(k) = [sampleratePermillage(k) jitterTimestamp(k)]^    

 timestampDiff(k) = t_bar(k)^ * w_bar(k) + v(k)             P(v) ~ (0,R)    (4.1.7)
 t_bar(k) = [recvTimeMsDiff(k) 1]^    

residual(k) = timestampDiff(k) - t_bar(k)^ * w_hat(k-1)                 (4.1.8)

4.2）TimestampExtrapolator模块计算期望接收时间

int64_t TimestampExtrapolator::ExtrapolateLocalTime(uint32_t timestamp90khz) {
  ReadLockScoped rl(*_rwLock);
  int64_t localTimeMs = 0;
  CheckForWrapArounds(timestamp90khz);
  double unwrapped_ts90khz =
      static_cast<double>(timestamp90khz) +
      _wrapArounds * ((static_cast<int64_t>(1) << 32) - 1);
  if (_packetCount == 0) {
    localTimeMs = -1;
  } else if (_packetCount < _startUpFilterDelayInPackets) {
    localTimeMs =
        _prevMs +
        static_cast<int64_t>(
            static_cast<double>(unwrapped_ts90khz - _prevUnwrappedTimestamp) /
                90.0 +
            0.5);
  } else {
    if (_w[0] < 1e-3) {
      localTimeMs = _startMs;
    } else {
      double timestampDiff =
          unwrapped_ts90khz - static_cast<double>(_firstTimestamp);
      localTimeMs = static_cast<int64_t>(static_cast<double>(_startMs) +
                                         (timestampDiff - _w[1]) / _w[0] + 0.5);
    }
  }
  return localTimeMs;
}

4.3）TimestampExtrapolator模块Kalman预测及校正

//参数tMs为当前帧实际接收时间
//参数ts90khz为当前帧的rtp时间戳
void TimestampExtrapolator::Update(int64_t tMs, uint32_t ts90khz) {
   _rwLock->AcquireLockExclusive();
  //1)第一帧初始赋值  
  if (tMs - _prevMs > 10e3) {//第一帧或者10秒钟内未收到任何完整的帧则重置
    // Ten seconds without a complete frame.
    // Reset the extrapolator
    _rwLock->ReleaseLockExclusive();
    Reset(tMs);
    _rwLock->AcquireLockExclusive();
  } else {
    _prevMs = tMs;
  } 
    
  //2)根据当前帧的本地接收时间计算detalRecvTimeMs(k)  
  // Remove offset to prevent badly scaled matrices
  // 将当前帧接收时间 - 第一帧的接收时间得当前帧和第一帧的本地接收时间差
  // 此处记为detalRecvTimeMs =  tMs - _startMs  
  int64_t recvTimeMsDiff = tMs - _startMs;

  CheckForWrapArounds(ts90khz);

  int64_t unwrapped_ts90khz =
      static_cast<int64_t>(ts90khz) +
      _wrapArounds * ((static_cast<int64_t>(1) << 32) - 1);

  if (_firstAfterReset) {//重置后赋值初值
    // Make an initial guess of the offset,
    // should be almost correct since tMs - _startMs
    // should about zero at this time.
    _w[1] = -_w[0] * tMs;
    _firstTimestamp = unwrapped_ts90khz;
    _firstAfterReset = false;
  }
  /*3)使用上一次最优估计计算网络残差为计算验估计做准备，对应5大核心公式的公式(4) 以及4.1.8
      用当前帧真实的rtp时间戳 - 第一帧的时间戳 - detalRecvTimeMs * _w[0] - _w[1]
      detalRecvTimeMs * _w[0](上一次的最优采样率) 得出detalRtpTimeStamp 
  */  
  double residual = (static_cast<double>(unwrapped_ts90khz) - _firstTimestamp) -
                    static_cast<double>(recvTimeMsDiff) * _w[0] - _w[1];
    
  if (DelayChangeDetection(residual) &&
      _packetCount >= _startUpFilterDelayInPackets) {
    // A sudden change of average network delay has been detected.
    // Force the filter to adjust its offset parameter by changing
    // the offset uncertainty. Don't do this during startup.
    _pP[1][1] = _pP11;
  }

  if (_prevUnwrappedTimestamp >= 0 &&
      unwrapped_ts90khz < _prevUnwrappedTimestamp) {
    // Drop reordered frames.
    _rwLock->ReleaseLockExclusive();
    return;
  }

  // T = [t(k) 1]';
  // that = T'*w;
  // K = P*T/(lambda + T'*P*T);
  // 4)计算卡尔曼增益  
  double K[2];
  // 对应5大公式,公式3中的分子部分  
  K[0] = _pP[0][0] * recvTimeMsDiff + _pP[0][1];
  K[1] = _pP[1][0] * recvTimeMsDiff + _pP[1][1];
  // 对应5大公式,公式3中的分母部分   
  double TPT = _lambda + recvTimeMsDiff * K[0] + K[1];
  K[0] /= TPT;
  K[1] /= TPT;
  //5) 根据最优卡尔曼因子进行校正，计算后验估计值  
  // w = w + K*(ts(k) - that);
  _w[0] = _w[0] + K[0] * residual;
  _w[1] = _w[1] + K[1] * residual;
  //6)更新误差协方差  
  // P = 1/lambda*(P - K*T'*P);
  double p00 =
      1 / _ * (_pP[0][0] - (K[0] * recvTimeMsDiff * _pP[0][0] + K[0] * _pP[1][0]));
  double p01 =
      1 / _lambda * (_pP[0][1] - (K[0] * recvTimeMsDiff * _pP[0][1] + K[0] * _pP[1][1]));
  _pP[1][0] =
      1 / _lambda * (_pP[1][0] - (K[1] * recvTimeMsDiff * _pP[0][0] + K[1] * _pP[1][0]));
  _pP[1][1] =
      1 / _lambda * (_pP[1][1] - (K[1] * recvTimeMsDiff * _pP[0][1] + K[1] * _pP[1][1]));
  _pP[0][0] = p00;
  _pP[0][1] = p01;
  _prevUnwrappedTimestamp = unwrapped_ts90khz;
  if (_packetCount < _startUpFilterDelayInPackets) {
    _packetCount++;
  }
  _rwLock->ReleaseLockExclusive();
}

expectRenderTime = expectRecvTime + actual_delay

5）计算期望渲染时间

void VCMTiming::SetJitterDelay(int jitter_delay_ms) {
  rtc::CritScope cs(&crit_sect_);
  if (jitter_delay_ms != jitter_delay_ms_) {
    jitter_delay_ms_ = jitter_delay_ms;
    // When in initial state, set current delay to minimum delay.
    if (current_delay_ms_ == 0) {
      current_delay_ms_ = jitter_delay_ms_;
    }
  }
}

void VCMTiming::UpdateCurrentDelay(int64_t render_time_ms,
                                   int64_t actual_decode_time_ms) {
  rtc::CritScope cs(&crit_sect_);
  uint32_t target_delay_ms = TargetDelayInternal();//目标延迟
  //计算实际延迟  
  int64_t delayed_ms =
      actual_decode_time_ms -
      (render_time_ms - RequiredDecodeTimeMs() - render_delay_ms_);
  if (delayed_ms < 0) {
    return;
  }
  if (current_delay_ms_ + delayed_ms <= target_delay_ms) {
    current_delay_ms_ += delayed_ms;
  } else {
    current_delay_ms_ = target_delay_ms;
  }
}

int VCMTiming::TargetDelayInternal() const {
  //计算出目标延迟 =  jitter_delay_ms_ + 解码耗时 +  渲染延迟
  return std::max(min_playout_delay_ms_,
                  jitter_delay_ms_ + RequiredDecodeTimeMs() + render_delay_ms_);
}

  current_delay_ms_ = 帧间抖动延时 + 解码耗时 + 渲染延迟 

expectRenderTime = expectRecvTime + actual_delay
                 = 期望接收时间 + 帧间抖动延时 + 解码耗时 + 渲染延迟

6）总结

上一篇 下一篇

猜你喜欢

热点阅读