Flutter视频渲染YUV数据转换BGRA
由于Flutter的Skia引擎只支持BGRA格式的数据。因此，在视频渲染时，需要进行数据转换，这里主要谈谈YUV420数据转BGRA数据。
注意：在数据转换过程中有用到libyuv库，可以自行导入：https://github.com/lemenkov/libyuv
首先，简要概述下视频格式
/*
kCVPixelFormatType类型的含义
kCVPixelFormatType_{长度|序列}{颜色空间}{Planar|BiPlanar}{VideoRange|FullRange}
Planar: 平面；BiPlanar：双平面
平面／双平面主要应用在yuv上。uv分开存储的为Planar，反之是BiPlanar。所以，kCVPixelFormatType_420YpCbCr8PlanarFullRange是420p，kCVPixelFormatType_420YpCbCr8BiPlanarFullRange是nv12.
*/
static OSType KVideoPixelFormatType = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
其次，数据转化、使用的整体流程
- (void)yuvPixelBufferWithData:(void *)dataFrame width:(int)w heigth:(int)h {
//YUV数据字符
unsignedchar*buffer = (unsignedchar*)dataFrame;
/******
// //数据是黑白的，采用下面libyuv的方式转换则正常
// CVPixelBufferRef getCroppedPixelBuffer = [self copyDataFromBuffer:buffer toYUVPixelBufferWithWidth:w Height:h];
******/
//使用libyuv库的 I420ToNV12() 方法转化数据正常。将 YUV420的char 转化为 CVPixelBuffer 类型的帧数据
CVPixelBufferRefgetOriginalPixelBuffer = [selfyuv420FrameToPixelBuffer:bufferwidth:wheight:h];
if(!getOriginalPixelBuffer) {
return;
}
//压缩数据
CMSampleBufferRefsampleBuffer = [selfpixelBufferToSampleBuffer:getOriginalPixelBuffer];
if(!sampleBuffer) {
return;
}
//通过libyuv库的方法，将sampleBuffer 转换为 32BGRA格式的CVPixelBuffer
CVPixelBufferReffinalPixelBuffer = [selfconvertVideoSmapleBufferToBGRAData:sampleBuffer];
if(!finalPixelBuffer) {
return;
}
CFRelease(getOriginalPixelBuffer);//释放
//实时更新帧数据，释放上一帧的内存，指向新一帧画面的内存
CFRelease(_target);
_target= finalPixelBuffer;
//实时回调，使得Flutter侧能够实时共享到最新的texture
_callback();
}
数据转换共分三步走
第一步：YUV420 数据转 CVPixelBufferRef(本质上还是YUV420数据)
/******************** YUV420 数据转 nv12的CVPixelBufferRef(本质上还是YUV420数据) **********************/
- (CVPixelBufferRef)yuv420FrameToPixelBuffer:(constunsignedchar*)yuv420Framewidth:(int)frameWidthheight:(int)frameHeight
{
if(yuv420Frame ==nil) {
returnNULL;
}
CVPixelBufferRefpixelBuffer = NULL;
NSDictionary *pixelBufferAttributes = [NSDictionary dictionaryWithObjectsAndKeys:[NSDictionary dictionary], (id)kCVPixelBufferIOSurfacePropertiesKey, nil];
//为YUV420格式的CVPixelBuffer分配内存
CVReturnresult =CVPixelBufferCreate(kCFAllocatorDefault, frameWidth, frameHeight,kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, (__bridge CFDictionaryRef)pixelBufferAttributes, &pixelBuffer);
if(result !=kCVReturnSuccess) {
NSLog(@"[yuv420FrameToPixelBuffer] Failed to create pixel buffer: %d", result);
returnNULL;
}
//加锁
result =CVPixelBufferLockBaseAddress(pixelBuffer,0);
if(result !=kCVReturnSuccess) {
CFRelease(pixelBuffer);
NSLog(@"[yuv420FrameToPixelBuffer] Failed to lock base address: %d", result);
return NULL;
}
//获取pixelBuffer中的Y数据
uint8 *dstY = (uint8 *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
intdstStrideY = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer,0);
//获取pixelBuffer中的UV数据
uint8* dstUV = (uint8*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
intdstStrideUV = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer,1);
UInt8*_planeData[3];
NSUInteger_stride[3];
uint8*_data = (uint8*)yuv420Frame;
_planeData[0] = _data;//y数据
_planeData[1] = _planeData[0] + frameWidth * frameHeight;//u
_planeData[2] = _planeData[1] + frameWidth * frameHeight /4;//v
_stride[0] = frameWidth;
_stride[1] = frameWidth >>1;
_stride[2] = frameWidth >>1;
//使用libyuv库的方法进行数据转换
intret =I420ToNV12(_planeData[0], (int)_stride[0],
_planeData[1], (int)_stride[1],
_planeData[2], (int)_stride[2],
dstY, dstStrideY,
dstUV, dstStrideUV,
frameWidth, frameHeight);
//解锁
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if(ret) {
NSLog(@"[yuv420FrameToPixelBuffer] Error converting yuv420 VideoFrame to NV12: %d", result);
CFRelease(pixelBuffer);
returnNULL;
}
returnpixelBuffer;
}/******************** YUV420 数据转 nv12的CVPixelBufferRef(本质上还是YUV420数据) **********************/
- (CVPixelBufferRef)yuv420FrameToPixelBuffer:(constunsignedchar*)yuv420Framewidth:(int)frameWidthheight:(int)frameHeight
{
if(yuv420Frame ==nil) {
returnNULL;
}
CVPixelBufferRefpixelBuffer = NULL;
NSDictionary *pixelBufferAttributes = [NSDictionary dictionaryWithObjectsAndKeys:[NSDictionary dictionary], (id)kCVPixelBufferIOSurfacePropertiesKey, nil];
//为YUV420格式的CVPixelBuffer分配内存
CVReturnresult =CVPixelBufferCreate(kCFAllocatorDefault, frameWidth, frameHeight,kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, (__bridge CFDictionaryRef)pixelBufferAttributes, &pixelBuffer);
if(result !=kCVReturnSuccess) {
NSLog(@"[yuv420FrameToPixelBuffer] Failed to create pixel buffer: %d", result);
returnNULL;
}
//加锁
result =CVPixelBufferLockBaseAddress(pixelBuffer,0);
if(result !=kCVReturnSuccess) {
CFRelease(pixelBuffer);
NSLog(@"[yuv420FrameToPixelBuffer] Failed to lock base address: %d", result);
return NULL;
}
//获取pixelBuffer中的Y数据
uint8 *dstY = (uint8 *)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 0);
intdstStrideY = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer,0);
//获取pixelBuffer中的UV数据
uint8* dstUV = (uint8*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, 1);
intdstStrideUV = (int)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer,1);
UInt8*_planeData[3];
NSUInteger_stride[3];
uint8*_data = (uint8*)yuv420Frame;
_planeData[0] = _data;//y数据
_planeData[1] = _planeData[0] + frameWidth * frameHeight;//u
_planeData[2] = _planeData[1] + frameWidth * frameHeight /4;//v
_stride[0] = frameWidth;
_stride[1] = frameWidth >>1;
_stride[2] = frameWidth >>1;
//使用libyuv库的方法进行数据转换
intret =I420ToNV12(_planeData[0], (int)_stride[0],
_planeData[1], (int)_stride[1],
_planeData[2], (int)_stride[2],
dstY, dstStrideY,
dstUV, dstStrideUV,
frameWidth, frameHeight);
//解锁
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
if(ret) {
NSLog(@"[yuv420FrameToPixelBuffer] Error converting yuv420 VideoFrame to NV12: %d", result);
CFRelease(pixelBuffer);
returnNULL;
}
returnpixelBuffer;
}
第二步：CVPixelBuffer数据压缩转换
//数据压缩
- (CMSampleBufferRef)pixelBufferToSampleBuffer:(CVPixelBufferRef)pixelBuffer {
CMSampleBufferRefsampleBuffer;
CMTime frameTime = CMTimeMakeWithSeconds([[NSDate date] timeIntervalSince1970], 1000000000);
CMSampleTimingInfotiming = {frameTime, frameTime,kCMTimeInvalid};
CMVideoFormatDescriptionRefvideoInfo = NULL ;
CMVideoFormatDescriptionCreateForImageBuffer(NULL , pixelBuffer, &videoInfo);
OSStatusstatus =CMSampleBufferCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer,true, NULL,NULL, videoInfo, &timing, &sampleBuffer);
//释放资源
CFRelease(pixelBuffer);
if(videoInfo) {
CFRelease(videoInfo);
}
if(status !=noErr) {
NSLog(@"[pixelBufferToSampleBuffer] Failed to create sample buffer with error %d.", ( int )status);
returnNULL;
}
return sampleBuffer;
}
第三步：CVPixelBuffer数据转换为BGRA格式
//转化_为kCVPixelFormatType_32BGRA类型数据（使Flutter的skia引擎可以绘制）
- (CVPixelBufferRef)convertVideoSmapleBufferToBGRAData:(CMSampleBufferRef)videoSample{
//CVPixelBufferRef是CVImageBufferRef的别名，两者操作几乎一致。
//获取CMSampleBuffer的图像地址
CVImageBufferRefpixelBuffer =CMSampleBufferGetImageBuffer(videoSample);
//VideoToolbox解码后的图像数据并不能直接给CPU访问，需先用CVPixelBufferLockBaseAddre()锁定地址才能从主存访问，否则调用CVPixelBufferGetBaseAddressOfPlane等函数则返回NULL或无效值。值得注意的是，CVPixelBufferLockBaseAddress自身的调用并不消耗多少性能，一般情况，锁定之后，往CVPixelBuffer拷贝内存才是相对耗时的操作，比如计算内存偏移。_
CVPixelBufferLockBaseAddress(pixelBuffer, 0);
//图像宽度（像素）
size_tpixelWidth =CVPixelBufferGetWidth(pixelBuffer);
//图像高度（像素）
size_tpixelHeight =CVPixelBufferGetHeight(pixelBuffer);
//获取CVImageBufferRef中的y数据
uint8_t*y_frame = (unsignedchar*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer,0);
//获取CMVImageBufferRef中的uv数据
uint8_t*uv_frame =(unsignedchar*)CVPixelBufferGetBaseAddressOfPlane(pixelBuffer,1);
// 创建一个空的32BGRA格式的CVPixelBufferRef
NSDictionary *pixelAttributes = @{(id)kCVPixelBufferIOSurfacePropertiesKey : @{}};
CVPixelBufferRefpixelBuffer1 =NULL;
CVReturnresult =CVPixelBufferCreate(kCFAllocatorDefault, pixelWidth, pixelHeight,kCVPixelFormatType_32BGRA, (__bridgeCFDictionaryRef)pixelAttributes, &pixelBuffer1);
if(result !=kCVReturnSuccess) {
NSLog(@"[convertVideoSmapleBufferToBGRAData] Unable to create cvpixelbuffer %d", result);
returnNULL;
}
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
result =CVPixelBufferLockBaseAddress(pixelBuffer1,0);
if(result !=kCVReturnSuccess) {
CFRelease(pixelBuffer1);
NSLog(@"[convertVideoSmapleBufferToBGRAData] Failed to lock base address: %d", result);
returnNULL;
}
// 得到新创建的CVPixelBufferRef中 rgb数据的首地址
uint8_t*rgb_data =CVPixelBufferGetBaseAddress(pixelBuffer1);
// 使用libyuv为rgb_data写入数据，将NV12转换为BGRA
int ret =NV12ToARGB(y_frame, pixelWidth, uv_frame, pixelWidth, rgb_data, pixelWidth *4, pixelWidth, pixelHeight);
if (ret) {
NSLog(@"[convertVideoSmapleBufferToBGRAData] Error converting NV12 VideoFrame to BGRA: %d", result);
CFRelease(pixelBuffer1);
returnNULL;
}
CVPixelBufferUnlockBaseAddress(pixelBuffer1, 0);
return pixelBuffer1;
}
