无人机H264视频流解码
近期开始学习H264的视频流解析。写此文章，作为记录，也梳理下相应的知识点。
1. 解码前我们先看一下H264的部分码流数据（红色部分用于帧类型识别）
图一
图中红色部分：
SPS帧：00000001 67 SPS帧数据：64001e ace80a03 d9 （仅供参考数据）
PPS帧：00000001 68 PPS帧数据：ee3cb0 （仅供参考数据）
I 帧：00000001 65 I 帧数据：8881 4017ff89 c0159b63 2b53a0a2 59a7cd02 5c82be32 39a6db81 efb24d1a 348af4b0 1323bbe1 33044f7e 8ccb735f f4a439a7 4db023df 1b4e0d7d d8b1a15d 56ecfa1a e02debc0 （这里只是展示部分数据）
I 帧数据为主要视频帧，视频图像保存在 I 帧，需要SPS帧和PPS帧配合才能解码。
开始解码准备工作：
需要引入的头文件：
#import<VideoToolbox/VideoToolbox.h>
需要定义的参数：
uint8_t *_sps;
NSInteger _spsSize;
uint8_t *_pps;
NSInteger _ppsSize;
VTDecompressionSessionRef _deocderSession;
CMVideoFormatDescriptionRef _decoderFormatDescription;
2. 解码前分离SPS、PPS、I帧数据
- (void) decodeVidoeUdp:(uint8_t *)udpBuf udpLen:(int)udpLen {
NSInteger PPSCommand;
NSInteger IFRCommand;
NSInteger spsCommand;
if (udpBuf[4] == 0x67) {
spsCommand = 0x67;
PPSCommand = 0x68;
IFRCommand = 0x65;
}
const uint8_t ppsStartCode[5] = {0, 0, 0, 1, PPSCommand};
const uint8_t IFrStartCode[5] = {0, 0, 0, 1, IFRCommand};
if (udpLen < 5) {
//printf("udp len is too short %d\n", udpLen);
return;
}
// 如果是sps帧, 需要将sps pps I帧分离
if (udpBuf[4] == spsCommand)
{
uint8_t *start = udpBuf, *end = udpBuf;
int count = 0;
while (count < udpLen) {
end++;
count++;
if (end[3] == 0x01) {
//如果找到pps帧，前面的就是sps数据。下面传入的是sps数据
if (memcmp(end, ppsStartCode, 5) == 0)
{
[self decodeFile:start frameLen:(int)(end - start)];
start = end;
}
//IDR帧
if (memcmp(end, IFrStartCode, 5) == 0) {
[self decodeFile:start frameLen:(int)(end - start)];
[self decodeFile:end frameLen:(int)(udpBuf + udpLen - end)];
break;
}
}
}
}else{
[self decodeFile:udpBuf frameLen:udpLen];
}
}
3. 准备解码数据
//初始化H264解码器
- (BOOL) initH264DecoderAndisValid:(BOOL)isVaild {
if(_deocderSession && isVaild) {
//printf("\nalready initH264Decoder\n");
return YES;
}
//printf("\ninitH264Decoder\n");
const uint8_t* const parameterSetPointers[2] = { _sps, _pps };
const size_t parameterSetSizes[2] = { _spsSize, _ppsSize };
OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault,2,//param countparameterSetPointers,parameterSetSizes,4, //nal start code size&_decoderFormatDescription);
if(status == noErr) {
CFDictionaryRef attrs = NULL;
const void *keys[] = { kCVPixelBufferPixelFormatTypeKey };
// kCVPixelFormatType_420YpCbCr8Planar is YUV420
// kCVPixelFormatType_420YpCbCr8BiPlanarFullRange is NV12
uint32_t v = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
const void *values[] = { CFNumberCreate(NULL, kCFNumberSInt32Type, &v) };
attrs = CFDictionaryCreate(NULL, keys, values, 1, NULL, NULL);
VTDecompressionOutputCallbackRecord callBackRecord
callBackRecord.decompressionOutputCallback = didDecompress
callBackRecord.decompressionOutputRefCon = NULL;
status = VTDecompressionSessionCreate(kCFAllocatorDefault,_decoderFormatDescription,NULL, attrs,&callBackRecord,&_deocderSession);
//NSLog(@"status:%d",status);
CFRelease(attrs);
attrs = NULL;
} else {
//NSLog(@"IOS8VT: reset decoder session failed status=%d", (int)status);
}
return YES;
}
//准备解码需要的数据
- (void) decodeFile:(uint8_t *)frameBuf frameLen:(int)frameLen {
uint8_t udpBuf_test[frameLen];
memcpy(udpBuf_test, frameBuf, frameLen);
const uint8_t KStartCode[4] = {0, 0, 0, 1};
__block CVPixelBufferRef pixelBuffer = NULL;
uint32_t nalSize = (uint32_t)(frameLen - 4);
uint8_t *pNalSize = (uint8_t*)(&nalSize);
//判断是否为帧头
if(memcmp(udpBuf_test, KStartCode, 4) != 0) {
//printf("\nframe buf is not a video frame\n");
return;
}
//将4个字节的帧头填充为帧长度,不包括帧头4个字节
udpBuf_test[0] = *(pNalSize + 3);
udpBuf_test[1] = *(pNalSize + 2);
udpBuf_test[2] = *(pNalSize + 1);
udpBuf_test[3] = *(pNalSize);
int nalType = udpBuf_test[4] & 0x1F;
switch (nalType) {
case 0x05:
//通过sps和pps初始化参数,并开始解码I帧
if([self initH264DecoderAndisValid:YES]) {
pixelBuffer = [self decode:udpBuf_test frameLen:frameLen];
}
break;
case 0x07:
if (_sps) {
break;
}
_spsSize = nalSize;
_sps = malloc(_spsSize);
memcpy(_sps, udpBuf_test + 4, _spsSize);
break;
case 0x08:
if (_pps) {
break;
}
_ppsSize = nalSize;
_pps = malloc(_ppsSize);
memcpy(_pps, udpBuf_test + 4, _ppsSize);
break;
case 0x01:
{
pixelBuffer = [self decode:udpBuf_test frameLen:frameLen];//这里得到视频中的图像数据
}
break;
default:
break;
}
//编码后返回的像素缓存入列到图像显示队列
if(pixelBuffer) {
dispatch_sync(dispatch_get_main_queue(), ^{
self.GLLayer.pixelBuffer = pixelBuffer;
});
CVPixelBufferRelease(pixelBuffer);
pixelBuffer = NULL;
}
}
4. 解码器解码（使用系统自带解码器解码）
- (CVPixelBufferRef) decode:(uint8_t *)frameBuf frameLen:(int)frameLen {
//NSLog(@"frameLen:%d",frameLen);
CVPixelBufferRef outputPixelBuffer = NULL;
CMBlockBufferRef blockBuffer = NULL;
//创建CMBlockBuffer
OSStatus status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault，(void*)frameBuf,frameLen,kCFAllocatorNull,NULL,0,frameLen,0,&blockBuffer);
if(status == kCMBlockBufferNoErr) {
CMSampleBufferRef sampleBuffer = NULL;
const size_t sampleSizeArray[] = {frameLen};
//创建CMSampleBuffer
status = CMSampleBufferCreateReady(kCFAllocatorDefault,blockBuffer,_decoderFormatDescription ,1, 0, NULL, 1, sampleSizeArray,&sampleBuffer);
if (status == kCMBlockBufferNoErr && sampleBuffer) {
VTDecodeFrameFlags flags = 0;
VTDecodeInfoFlags flagOut = 0;
if (_deocderSession == NULL || sampleBuffer == NULL) {
//printf("null pointer %p %p", _deocderSession, sampleBuffer);
return NULL;
}
//开始解码，解码后存放在outputPixelBuffer
OSStatus decodeStatus = VTDecompressionSessionDecodeFrame(_deocderSession,sampleBuffer,flags,&outputPixelBuffer,&flagOut);
if(decodeStatus == kVTInvalidSessionErr) {
[self initH264DecoderAndisValid:NO];
//NSLog(@"IOS8VT: Invalid session, reset decoder session");
} else if(decodeStatus == kVTVideoDecoderBadDataErr) {
//NSLog(@"IOS8VT: decode failed status=%d(Bad data)", (int)decodeStatus);
} else if(decodeStatus != noErr) {
[self clearH264Deocder];
[self initH264DecoderAndisValid:NO];
//NSLog(@"IOS8VT: decode failed status=%d", (int)decodeStatus);
return nil;
}
CFRelease(sampleBuffer);
sampleBuffer = NULL;
}
CFRelease(blockBuffer);
blockBuffer = NULL;
}
return outputPixelBuffer;
}
5. 释放解码器
- (void) clearH264Deocder {
[self.lock lock];
if(_deocderSession) {
VTDecompressionSessionInvalidate(_deocderSession);
if (_deocderSession == nil) {
[self.lock unlock];
return;
}
CFRelease(_deocderSession);
_deocderSession = NULL;
}
if(_decoderFormatDescription) {
CFRelease(_decoderFormatDescription);
_decoderFormatDescription = NULL;
}
free(_sps);
free(_pps);
_sps = _pps = NULL;
_spsSize = _ppsSize = 0;
[self.lock unlock];
}
解码流程大致就是这5个步骤，刚接触时心里是懵逼的，接触久了慢慢就熟悉了。本人也是新手一枚，如果有错误的地方还请指正。不胜感激。
