webrtc

WebRtc Video Receiver(五)-设置参考帧

2020-10-12  本文已影响0人  JeffreyLau

1）前言

void RtpVideoStreamReceiver2::OnAssembledFrame(
    std::unique_ptr<video_coding::RtpFrameObject> frame) {
  RTC_DCHECK_RUN_ON(&worker_task_checker_);
  RTC_DCHECK(frame);
  .....
  //该模块默认未开启，新特性值得研究，顾名思义为丢包通知控制模块    
  // 可通过WebRTC-RtcpLossNotification/Enable开启，但是默认只支持VP8
  // SDP需要实现goog-lntf feedback    
  if (loss_notification_controller_ && descriptor) {
    loss_notification_controller_->OnAssembledFrame(
        frame->first_seq_num(), descriptor->frame_id,
        absl::c_linear_search(descriptor->decode_target_indications,
                              DecodeTargetIndication::kDiscardable),
        descriptor->dependencies);
  }         
  // If frames arrive before a key frame, they would not be decodable.
  // In that case, request a key frame ASAP.
  if (!has_received_frame_) {
    if (frame->FrameType() != VideoFrameType::kVideoFrameKey) {
      // |loss_notification_controller_|, if present, would have already
      // requested a key frame when the first packet for the non-key frame
      // had arrived, so no need to replicate the request.
      if (!loss_notification_controller_) {
        RequestKeyFrame();
      }
    }
    has_received_frame_ = true;
  }

  // Reset |reference_finder_| if |frame| is new and the codec have changed.
  if (current_codec_) {
    //每帧之间的时间戳不一样，当前帧的时间戳大于前一帧的时间戳(未环绕的情况下)  
    bool frame_is_newer =
        AheadOf(frame->Timestamp(), last_assembled_frame_rtp_timestamp_);

    if (frame->codec_type() != current_codec_) {
      if (frame_is_newer) {
        // When we reset the |reference_finder_| we don't want new picture ids
        // to overlap with old picture ids. To ensure that doesn't happen we
        // start from the |last_completed_picture_id_| and add an offset in case
        // of reordering.
        reference_finder_ =
            std::make_unique<video_coding::RtpFrameReferenceFinder>(
                this, last_completed_picture_id_ +
                          std::numeric_limits<uint16_t>::max());
        current_codec_ = frame->codec_type();
      } else {
        // Old frame from before the codec switch, discard it.
        return;
      }
    }

    if (frame_is_newer) {
      last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
    }
  } else {
    current_codec_ = frame->codec_type();
    last_assembled_frame_rtp_timestamp_ = frame->Timestamp();
  }

  if (buffered_frame_decryptor_ != nullptr) {
    buffered_frame_decryptor_->ManageEncryptedFrame(std::move(frame));
  } else if (frame_transformer_delegate_) {
    frame_transformer_delegate_->TransformFrame(std::move(frame));
  } else {
    reference_finder_->ManageFrame(std::move(frame));
  }
}

2）ManageFrame工作流程

void RtpFrameReferenceFinder::ManageFrame(
    std::unique_ptr<RtpFrameObject> frame) {
  // If we have cleared past this frame, drop it.
  if (cleared_to_seq_num_ != -1 &&
      AheadOf<uint16_t>(cleared_to_seq_num_, frame->first_seq_num())) {
    return;
  }
  
  FrameDecision decision = ManageFrameInternal(frame.get());

  switch (decision) {
    case kStash:
      if (stashed_frames_.size() > kMaxStashedFrames)
        stashed_frames_.pop_back();
      stashed_frames_.push_front(std::move(frame));
      break;
    case kHandOff:
      HandOffFrame(std::move(frame));
      RetryStashedFrames();
      break;
    case kDrop:
      break;
  }
}

RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFrameInternal(RtpFrameObject* frame) {
  ........
  switch (frame->codec_type()) {
    case kVideoCodecVP8:
      return ManageFrameVp8(frame);
    case kVideoCodecVP9:
      return ManageFrameVp9(frame);
    case kVideoCodecGeneric:
      if (auto* generic_header = absl::get_if<RTPVideoHeaderLegacyGeneric>(
              &frame->GetRtpVideoHeader().video_type_header)) {
        return ManageFramePidOrSeqNum(frame, generic_header->picture_id);
      }
      ABSL_FALLTHROUGH_INTENDED;
    default:
      return ManageFramePidOrSeqNum(frame, kNoPictureId);
  }
}

3）ManageFramePidOrSeqNum设置参考帧

RtpFrameReferenceFinder::FrameDecision RtpFrameReferenceFinder::ManageFrameH264(
    RtpFrameObject* frame) {
  const FrameMarking& rtp_frame_marking = frame->GetFrameMarking();

  uint8_t tid = rtp_frame_marking.temporal_id;
  bool blSync = rtp_frame_marking.base_layer_sync;
  /*android 硬编的情况收到的tid位0xff,传入的kNoPictureId=-1，这是h264的特性*/ 
  if (tid == kNoTemporalIdx)
    return ManageFramePidOrSeqNum(std::move(frame), kNoPictureId);
  ....  
}

RtpFrameReferenceFinder::FrameDecision
RtpFrameReferenceFinder::ManageFramePidOrSeqNum(RtpFrameObject* frame,
                                                int picture_id) {
  // If |picture_id| is specified then we use that to set the frame references,
  // otherwise we use sequence number.
  // 1）确保非h264帧gop内维护的帧的连续性  
  if (picture_id != kNoPictureId) {
    frame->id.picture_id = unwrapper_.Unwrap(picture_id);
    frame->num_references =
        frame->frame_type() == VideoFrameType::kVideoFrameKey ? 0 : 1;
    frame->references[0] = frame->id.picture_id - 1;
    return kHandOff;
  }
    
  //2）判断是否为关键帧,其中frame_type在组帧的时候进行设置的
  if (frame->frame_type() == VideoFrameType::kVideoFrameKey) {
    last_seq_num_gop_.insert(std::make_pair(
        frame->last_seq_num(),//当前gop最后一个包的seq为key
        std::make_pair(frame->last_seq_num(), frame->last_seq_num())));
  }
  //3）如果到此为止还没有收到一帧关键帧，则存储该帧
  // We have received a frame but not yet a keyframe, stash this frame.
  if (last_seq_num_gop_.empty()) 
    return kStash;
    
  // Clean up info for old keyframes but make sure to keep info
  // for the last keyframe.
  // 4）清除老的gop frame->last_seq_num() - 100之前的所有都清除掉，但至少确保有一个。
  auto clean_to = last_seq_num_gop_.lower_bound(frame->last_seq_num() - 100);
  for (auto it = last_seq_num_gop_.begin();
       it != clean_to && last_seq_num_gop_.size() > 1;) {
    it = last_seq_num_gop_.erase(it);
  }
    
  // Find the last sequence number of the last frame for the keyframe
  // that this frame indirectly references.
  // 函数能走到这一步，gop 容器中是一定有存值的  
  //5.1) 如果关键帧的序号是大于该帧的序号的（未环绕的情况），那么该帧需要丢弃掉。  
  // 假设last_seq_num_gop_中存的是34号包，而本次来的帧的序号是10~16（非关键帧）。
  //5.2) 还有一种情况假设当前帧就是关键帧frame->last_seq_num()=34，假设事先last_seq_num_gop_存的是56号seq,由last_seq_num_gop_定义的排序规则，34号包被插入的时候会在头部，最终下面的条件依然成立。  
  auto seq_num_it = last_seq_num_gop_.upper_bound(frame->last_seq_num());
  if (seq_num_it == last_seq_num_gop_.begin()) {
    RTC_LOG(LS_WARNING) << "Generic frame with packet range ["
                        << frame->first_seq_num() << ", "
                        << frame->last_seq_num()
                        << "] has no GoP, dropping frame.";
    return kDrop;
  }
  //如果上述条件不成立这里则返回last_seq_num_gop_最后一个元素对应的迭代器
  //如果当前帧为关键帧的话那么seq_num_it为last_seq_num_gop_.end(),进行--操作后旧对应了最后一个关键帧  
  seq_num_it--;

  // Make sure the packet sequence numbers are continuous, otherwise stash
  // this frame.
  // 6) 该步用来判断该帧和上一帧的连续性 
  // last_picture_id_gop得到的是当前gop所维护的当前帧的上一帧（前向参考帧）的最后一个包的seq。  
  uint16_t last_picture_id_gop = seq_num_it->second.first;
  // last_picture_id_with_padding_gop得到的也是上一帧的最后一个包的seq。
  // 当前GOP的最新包的序列号，可能是last_picture_id_gop, 也可能是填充包.  
  uint16_t last_picture_id_with_padding_gop = seq_num_it->second.second;
  // 非关键帧判断seq连续性，  
  if (frame->frame_type() == VideoFrameType::kVideoFrameDelta) {
    //得到上一帧最后一个包的seq,当前帧的第一个包的seq -1 得到上一帧的最后一个seq  
    uint16_t prev_seq_num = frame->first_seq_num() - 1;
    // 如果不相等说明不连续，如果正常未丢包的情况下是一定会相等的。  
    if (prev_seq_num != last_picture_id_with_padding_gop)
      return kStash;
  }
  //检查当前帧最后一个seq是否大于所属gop 关键帧的最后一个seq
  RTC_DCHECK(AheadOrAt(frame->last_seq_num(), seq_num_it->first));

  // Since keyframes can cause reordering we can't simply assign the
  // picture id according to some incrementing counter.
  //7) 给RtpFrameObject的id.picture_id赋值
  // 如果为关键帧num_references为false,否则为true  
  frame->id.picture_id = frame->last_seq_num();
  frame->num_references =
      frame->frame_type() == VideoFrameType::kVideoFrameDelta;
  //上一帧最后一个包号  
  frame->references[0] = rtp_seq_num_unwrapper_.Unwrap(last_picture_id_gop);
  //这一步确保第6步的逻辑能跑通，否则第6不逻辑是跑不通的last_picture_id_表示的是当前帧的上一个关键帧的最后一个包的seq,frame->id.picture_id为当前帧的最后一个包的seq，正常情况AheadOf函数是会返回true的。  
  if (AheadOf<uint16_t>(frame->id.picture_id, last_picture_id_gop)) {
    //这里修改了容器last_seq_num_gop_对应关键帧的second变量，将当前帧最后一个包号的seq 赋值给他们 
    //正因为有这个操作，第6步才能顺利跑通  
    seq_num_it->second.first = frame->id.picture_id;
    seq_num_it->second.second = frame->id.picture_id;
  }

  last_picture_id_ = frame->id.picture_id;
  //更新填充包状态  
  UpdateLastPictureIdWithPadding(frame->id.picture_id);
  frame->id.picture_id = rtp_seq_num_unwrapper_.Unwrap(frame->id.picture_id);
  return kHandOff;
}

4) UpdateLastPictureIdWithPadding更新填充包状态

void RtpFrameReferenceFinder::UpdateLastPictureIdWithPadding(uint16_t seq_num) {
  //取第一个大于seq_num的对应的gop   
  auto gop_seq_num_it = last_seq_num_gop_.upper_bound(seq_num);

  // If this padding packet "belongs" to a group of pictures that we don't track
  // anymore, do nothing.
  if (gop_seq_num_it == last_seq_num_gop_.begin())
    return;
  --gop_seq_num_it;

  // Calculate the next contiuous sequence number and search for it in
  // the padding packets we have stashed.
  uint16_t next_seq_num_with_padding = gop_seq_num_it->second.second + 1;
  auto padding_seq_num_it =
      stashed_padding_.lower_bound(next_seq_num_with_padding);

  // While there still are padding packets and those padding packets are
  // continuous, then advance the "last-picture-id-with-padding" and remove
  // the stashed padding packet.
  while (padding_seq_num_it != stashed_padding_.end() &&
         *padding_seq_num_it == next_seq_num_with_padding) {
    gop_seq_num_it->second.second = next_seq_num_with_padding;
    ++next_seq_num_with_padding;
    padding_seq_num_it = stashed_padding_.erase(padding_seq_num_it);
  }

  // In the case where the stream has been continuous without any new keyframes
  // for a while there is a risk that new frames will appear to be older than
  // the keyframe they belong to due to wrapping sequence number. In order
  // to prevent this we advance the picture id of the keyframe every so often.
  if (ForwardDiff(gop_seq_num_it->first, seq_num) > 10000) {
    auto save = gop_seq_num_it->second;
    last_seq_num_gop_.clear();
    last_seq_num_gop_[seq_num] = save;
  }
}

5) ManageFrame函数业务处理

void RtpFrameReferenceFinder::ManageFrame(
    std::unique_ptr<RtpFrameObject> frame) {
  .....
  FrameDecision decision = ManageFrameInternal(frame.get());

  switch (decision) {
    case kStash:
      if (stashed_frames_.size() > kMaxStashedFrames)//最大100
        stashed_frames_.pop_back();
      stashed_frames_.push_front(std::move(frame));
      break;
    case kHandOff:
      HandOffFrame(std::move(frame));
      RetryStashedFrames();
      break;
    case kDrop:
      break;
  }
}

void RtpFrameReferenceFinder::HandOffFrame(
    std::unique_ptr<RtpFrameObject> frame) {
  //picture_id_offset_为0  
  frame->id.picture_id += picture_id_offset_;
  for (size_t i = 0; i < frame->num_references; ++i) {
    frame->references[i] += picture_id_offset_;
  }
  frame_callback_->OnCompleteFrame(std::move(frame));
}

void RtpFrameReferenceFinder::RetryStashedFrames() {
  bool complete_frame = false;
  do {
    complete_frame = false;
    for (auto frame_it = stashed_frames_.begin();
         frame_it != stashed_frames_.end();) {
      FrameDecision decision = ManageFrameInternal(frame_it->get());

      switch (decision) {
        case kStash:
          ++frame_it;
          break;
        case kHandOff:
          complete_frame = true;
          HandOffFrame(std::move(*frame_it));
          RTC_FALLTHROUGH();
        case kDrop:
          frame_it = stashed_frames_.erase(frame_it);
      }
    }
  } while (complete_frame);
}

6）总结

上一篇下一篇

猜你喜欢

热点阅读