iOS | H.264编解码(VideoToolbox硬编解码)
2018-12-04 本文已影响0人
Leon_520
介绍
iOS 8.0 之后,苹果开放了硬解码和硬解码的API。VideoToolbox 是一套纯C语言API。其中包含了很多C语言函数; VideoToolbox是一个低级框架，可直接访问硬件编码器和解码器。它提供视频压缩和解压缩服务,本文主要针对H.264硬编码来进行编解码说明.关于H.264相关知识请参考H.264介绍&编码原理本文不做过多解释!
编码
-
硬解码：用GPU来解码，减少CPU运算- 优点：播放流畅、低功耗，解码速度快
- 缺点：兼容不好
-
软解码：用CPU来解码,比如(ffmpeg)- 优点：兼容好
- 缺点：加大CPU负担，耗电增加、没有硬解码流畅，解码速度相对慢
VideoToolbox编码:
1. 首先需要导入#import <VideoToolbox/VideoToolbox.h>
2. 初始化编码会话
@property (nonatomic, assign) VTCompressionSessionRef compressionSession;
// 初始化编码器
- (void)setupVideoSession {
// 1.用于记录当前是第几帧数据
self.frameID = 0;
// 2.录制视频的宽度&高度,根据实际需求修改
int width = 720;
int height = 1280;
// 3.创建CompressionSession对象,该对象用于对画面进行编码
OSStatus status = VTCompressionSessionCreate(NULL, // 会话的分配器。传递NULL以使用默认分配器。
width, // 帧的宽度，以像素为单位。
height, // 帧的高度，以像素为单位。
kCMVideoCodecType_H264, // 编解码器的类型,表示使用h.264进行编码
NULL, // 指定必须使用的特定视频编码器。传递NULL让视频工具箱选择编码器。
NULL, // 源像素缓冲区所需的属性，用于创建像素缓冲池。如果不希望视频工具箱为您创建一个，请传递NULL
NULL, // 压缩数据的分配器。传递NULL以使用默认分配器。
didCompressH264, // 当一次编码结束会在该函数进行回调,可以在该函数中将数据,写入文件中
(__bridge void *)(self), // outputCallbackRefCon
&_compressionSession); // 指向一个变量以接收的压缩会话。
if (status != 0){
NSLog(@"H264: session 创建失败");
return ;
}
// 4.设置实时编码输出（直播必然是实时输出,否则会有延迟）
VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
VTSessionSetProperty(_compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
// 5.设置关键帧（GOPsize)间隔
int frameInterval = 60;
CFNumberRef frameIntervalRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &frameInterval);
VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, frameIntervalRef);
// 6.设置期望帧率(每秒多少帧,如果帧率过低,会造成画面卡顿)
int fps = 24;
CFNumberRef fpsRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &fps);
VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, fpsRef);
// 7.设置码率(码率: 编码效率, 码率越高,则画面越清晰, 如果码率较低会引起马赛克 --> 码率高有利于还原原始画面,但是也不利于传输)
int bitRate = width * height * 3 * 4 * 8;
CFNumberRef bitRateRef = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &bitRate);
VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_AverageBitRate, bitRateRef);
// 8.设置码率，均值，单位是byte 这是一个算法
NSArray *limit = @[@(bitRate * 1.5/8), @(1)];
VTSessionSetProperty(self.compressionSession, kVTCompressionPropertyKey_DataRateLimits, (__bridge CFArrayRef)limit);
// 9.基本设置结束, 准备进行编码
VTCompressionSessionPrepareToEncodeFrames(_compressionSession);
}
3. 编码完成回调函数
// 编码完成回调
void didCompressH264(void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer) {
// 1.判断状态是否等于没有错误
if (status != noErr) {
return;
}
if (!CMSampleBufferDataIsReady(sampleBuffer)) {
NSLog(@"didCompressH264 data is not ready ");
return;
}
// 2.根据传入的参数获取对象
VideoH264EnCode* encoder = (__bridge VideoH264EnCode*)outputCallbackRefCon;
// 3.判断是否是关键帧
bool isKeyframe = !CFDictionaryContainsKey( (CFArrayGetValueAtIndex(CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true), 0)), kCMSampleAttachmentKey_NotSync);
// 判断当前帧是否为关键帧
// 获取sps & pps数据
if (isKeyframe)
{
// 获取编码后的信息（存储于CMFormatDescriptionRef中）
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);
// 获取SPS信息
size_t sparameterSetSize, sparameterSetCount;
const uint8_t *sparameterSet;
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &sparameterSet, &sparameterSetSize, &sparameterSetCount, 0 );
// 获取PPS信息
size_t pparameterSetSize, pparameterSetCount;
const uint8_t *pparameterSet;
CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 1, &pparameterSet, &pparameterSetSize, &pparameterSetCount, 0 );
// 装sps/pps转成NSData
NSData *sps = [NSData dataWithBytes:sparameterSet length:sparameterSetSize];
NSData *pps = [NSData dataWithBytes:pparameterSet length:pparameterSetSize];
// 写入文件
[encoder gotSpsPps:sps pps:pps];
}
// 获取数据块
CMBlockBufferRef dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
size_t length, totalLength;
char *dataPointer;
OSStatus statusCodeRet = CMBlockBufferGetDataPointer(dataBuffer, 0, &length, &totalLength, &dataPointer);
if (statusCodeRet == noErr) {
size_t bufferOffset = 0;
static const int AVCCHeaderLength = 4; // 返回的nalu数据前四个字节不是0001的startcode，而是大端模式的帧长度length
// 循环获取nalu数据
while (bufferOffset < totalLength - AVCCHeaderLength) {
uint32_t NALUnitLength = 0;
// Read the NAL unit length
memcpy(&NALUnitLength, dataPointer + bufferOffset, AVCCHeaderLength);
// 从大端转系统端
NALUnitLength = CFSwapInt32BigToHost(NALUnitLength);
NSData* data = [[NSData alloc] initWithBytes:(dataPointer + bufferOffset + AVCCHeaderLength) length:NALUnitLength];
[encoder gotEncodedData:data isKeyFrame:isKeyframe];
// 移动到写一个块，转成NALU单元
// Move to the next NAL unit in the block buffer
bufferOffset += AVCCHeaderLength + NALUnitLength;
}
}
}
4. 获取SPS/PPS,以及I,P,B 帧数据,并将其通过 block 回调
// 获取 sps 以及 pps,并进行StartCode
- (void)gotSpsPps:(NSData*)sps pps:(NSData*)pps{
// 拼接NALU的 StartCode,默认规定使用 00000001
const char bytes[] = "\x00\x00\x00\x01";
size_t length = (sizeof bytes) - 1;
NSData *ByteHeader = [NSData dataWithBytes:bytes length:length];
NSMutableData *h264Data = [[NSMutableData alloc] init];
[h264Data appendData:ByteHeader];
[h264Data appendData:sps];
if (self.h264DataBlock) {
self.h264DataBlock(h264Data);
}
[h264Data resetBytesInRange:NSMakeRange(0, [h264Data length])];
[h264Data setLength:0];
[h264Data appendData:ByteHeader];
[h264Data appendData:pps];
if (self.h264DataBlock) {
self.h264DataBlock(h264Data);
}
}
- (void)gotEncodedData:(NSData*)data isKeyFrame:(BOOL)isKeyFrame{
const char bytes[] = "\x00\x00\x00\x01";
size_t length = (sizeof bytes) - 1; //string literals have implicit trailing '\0'
NSData *ByteHeader = [NSData dataWithBytes:bytes length:length];
NSMutableData *h264Data = [[NSMutableData alloc] init];
[h264Data appendData:ByteHeader];
[h264Data appendData:data];
if (self.h264DataBlock) {
self.h264DataBlock(h264Data);
}
}
5. 通过传入原始帧数据进行调用并回调
// 将 sampleBuffer(摄像头捕捉数据,原始帧数据) 编码为H.264
- (void)encodeSampleBuffer:(CMSampleBufferRef)sampleBuffer H264DataBlock:(void (^)(NSData * _Nonnull))h264DataBlock{
if (!self.compressionSession) {
return;
}
// 1.保存 block 块
self.h264DataBlock = h264DataBlock;
// 2.将sampleBuffer转成imageBuffer
CVImageBufferRef imageBuffer = (CVImageBufferRef)CMSampleBufferGetImageBuffer(sampleBuffer);
// 3.根据当前的帧数,创建CMTime的时间
CMTime presentationTimeStamp = CMTimeMake(self.frameID++, 1000);
VTEncodeInfoFlags flags;
// 4.开始编码该帧数据
OSStatus statusCode = VTCompressionSessionEncodeFrame(
self.compressionSession,
imageBuffer,
presentationTimeStamp,
kCMTimeInvalid,
NULL,
(__bridge void * _Nullable)(self),
&flags
);
if (statusCode != noErr) {
NSLog(@"H264: VTCompressionSessionEncodeFrame failed with %d", (int)statusCode);
VTCompressionSessionInvalidate(self.compressionSession);
CFRelease(self.compressionSession);
self.compressionSession = NULL;
return;
}
}
通过上述几个方法,可以完成原始帧数据的H.264编码工作,并将其回调给调用者, 具体如何拼接以及使用,根据自身项目需求来进行使用
VideoToolbox解码:
解码与编码正好相反,在拿到H.264的每一帧数据后进行解码操作,最后获取 原始帧数据进行展示
1. 初始化解码器
- (BOOL)initH264Decoder {
if(_deocderSession) {
return YES;
}
const uint8_t* const parameterSetPointers[2] = { _sps, _pps };
const size_t parameterSetSizes[2] = { _spsSize, _ppsSize };
OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault,
2, //param count
parameterSetPointers,
parameterSetSizes,
4, //nal start code size
&_decoderFormatDescription);
if(status == noErr) {
NSDictionary* destinationPixelBufferAttributes = @{
(id)kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithInt:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange], //硬解必须是 kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange 或者是kCVPixelFormatType_420YpCbCr8Planar
//这里款高和编码反的
(id)kCVPixelBufferOpenGLCompatibilityKey : [NSNumber numberWithBool:YES]
};
VTDecompressionOutputCallbackRecord callBackRecord;
callBackRecord.decompressionOutputCallback = didDecompress;
callBackRecord.decompressionOutputRefCon = (__bridge void *)self;
status = VTDecompressionSessionCreate(kCFAllocatorDefault,
_decoderFormatDescription,
NULL,
(__bridge CFDictionaryRef)destinationPixelBufferAttributes,
&callBackRecord,
&_deocderSession);
VTSessionSetProperty(_deocderSession, kVTDecompressionPropertyKey_ThreadCount, (__bridge CFTypeRef)[NSNumber numberWithInt:1]);
VTSessionSetProperty(_deocderSession, kVTDecompressionPropertyKey_RealTime, kCFBooleanTrue);
} else {
NSLog(@"IOS8VT: reset decoder session failed status=%d", (int)status);
}
return YES;
}
2. 解码操作
// 解码操作,外部调用
- (void)decodeNalu:(uint8_t *)frame size:(uint32_t) frameSize{
int nalu_type = (frame[4] & 0x1F);
CVPixelBufferRef pixelBuffer = NULL;
uint32_t nalSize = (uint32_t)(frameSize - 4);
uint8_t *pNalSize = (uint8_t*)(&nalSize);
frame[0] = *(pNalSize + 3);
frame[1] = *(pNalSize + 2);
frame[2] = *(pNalSize + 1);
frame[3] = *(pNalSize);
//传输的时候。关键帧不能丢数据 否则绿屏 B/P可以丢 这样会卡顿
switch (nalu_type)
{
case 0x05:
// 关键帧
if([self initH264Decoder])
{
pixelBuffer = [self decode:frame withSize:frameSize];
}
break;
case 0x07:
// sps
_spsSize = frameSize - 4;
_sps = malloc(_spsSize);
memcpy(_sps, &frame[4], _spsSize);
break;
case 0x08:
{
// pps
_ppsSize = frameSize - 4;
_pps = malloc(_ppsSize);
memcpy(_pps, &frame[4], _ppsSize);
break;
}
default:
{
// B/P其他帧
if([self initH264Decoder]){
pixelBuffer = [self decode:frame withSize:frameSize];
}
break;
}
}
}
- (CVPixelBufferRef)decode:(uint8_t *)frame withSize:(uint32_t)frameSize{
CVPixelBufferRef outputPixelBuffer = NULL;
CMBlockBufferRef blockBuffer = NULL;
OSStatus status = CMBlockBufferCreateWithMemoryBlock(NULL,
(void *)frame,
frameSize,
kCFAllocatorNull,
NULL,
0,
frameSize,
FALSE,
&blockBuffer);
if(status == kCMBlockBufferNoErr) {
CMSampleBufferRef sampleBuffer = NULL;
const size_t sampleSizeArray[] = {frameSize};
status = CMSampleBufferCreateReady(kCFAllocatorDefault,
blockBuffer,
_decoderFormatDescription ,
1, 0, NULL, 1, sampleSizeArray,
&sampleBuffer);
if (status == kCMBlockBufferNoErr && sampleBuffer) {
VTDecodeFrameFlags flags = 0;
VTDecodeInfoFlags flagOut = 0;
OSStatus decodeStatus = VTDecompressionSessionDecodeFrame(_deocderSession,
sampleBuffer,
flags,
&outputPixelBuffer,
&flagOut);
if(decodeStatus == kVTInvalidSessionErr) {
NSLog(@"IOS8VT: Invalid session, reset decoder session");
} else if(decodeStatus == kVTVideoDecoderBadDataErr) {
NSLog(@"IOS8VT: decode failed status=%d(Bad data)", (int)decodeStatus);
} else if(decodeStatus != noErr) {
NSLog(@"IOS8VT: decode failed status=%d", (int)decodeStatus);
}
CFRelease(sampleBuffer);
}
CFRelease(blockBuffer);
}
return outputPixelBuffer;
}
3. 解码完成回调
// 解码回调函数
static void didDecompress( void *decompressionOutputRefCon, void *sourceFrameRefCon, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef pixelBuffer, CMTime presentationTimeStamp, CMTime presentationDuration ){
CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *)sourceFrameRefCon;
*outputPixelBuffer = CVPixelBufferRetain(pixelBuffer);
VideoH264Decoder *decoder = (__bridge VideoH264Decoder *)decompressionOutputRefCon;
if ([decoder.delegate respondsToSelector:@selector(decoder:didDecodingFrame:)]) {
[decoder.delegate decoder: decoder didDecodingFrame:pixelBuffer];
}
}
4. 通过 OpenGL 进行 帧数据展示
代码就不贴出来了,可以通过 demo 进行查看.
本文主要通过 VideoToolbox 对 iPhone 手机摄像头拍摄的视频流进行 编码和解码,并进行展示,仅仅提供了基本的编解码功能,具体在项目中如何使用还要根据自身项目来定,关于视频流传输,可以参考 Socket & CocoaAsyncSocket介绍与使用,以及如何处理粘包等问题;
更详细使用请查看: VideoToolbox官方说明文档
