IJKPlayer实时录制网络流

2021-12-08  本文已影响0人  萤火虫_629e

一、概要说明

网络流播放时,IJKPlayer新增录制mp4功能,因为是实时流,需要逐帧编码,当然也可以在播放器未解码前,逐帧写入到文件,但是不能确定读到的是什么格式的流,直接逐帧写入到文件怕不能使用,故采用解码后的流逐帧编码再写入的方式录制视频.

二、概要设计

三、音视频数据回调

需要实现这个功能,重点修改代码在ffplayer里面

1、在视频解码处把视频原始数据及pts回调出去,在queue_picture方法中,搜索SDL_VoutFillFrameYUVOverlay,在下面添加:

  // FIXME: set swscale options

if (SDL_VoutFillFrameYUVOverlay(vp->bmp, src_frame) < 0) {

    av_log(NULL, AV_LOG_FATAL, "Cannot initialize the conversion context\n");

    exit(1);

}

//新增代码

if (ffp -> videotoolbox) {

  // TODO edit

    ffp_pixelbuffer_lock(ffp);

    ffp->szt_pixelbuffer = SDL_VoutOverlayVideoToolBox_GetCVPixelBufferRef(vp->bmp); // picture->opaque;

    if (s_pixel_buff_callback)

//                ffp->stat.vdps

        s_pixel_buff_callback(ffp->inject_opaque, ffp->szt_pixelbuffer, vp->pts*1000*1000);

    ffp_pixelbuffer_unlock(ffp);

    if (!ffp->szt_pixelbuffer) {

        ALOGE("nil pixelBuffer in overlay\n");

    }

}

2、在音频解码后回调音频PCM数据,ffmpeg中回调的音频是32位float,但是试了下在iOS中不能正常编码,故先转格式为16位Int型,在int audio_thread(void *arg)中,在do{}while循环中,增加如下代码:

if (is->swr_ctx) {

 uint8_t *targetData[1];

 int len = 5760*2; //这里这么写定死长度可能会有问题

 targetData[0] = (uint8_t *)malloc(len);

 int size = audio_swr_resampling_audio(is->swr_ctx, context, frame, targetData);

 tb = (AVRational){1, frame->sample_rate};

 ffp_pcm_lock(ffp);

 int pts = (frame->pts == AV_NOPTS_VALUE) ? NAN : frame->pts * av_q2d(tb);

 uint8_t *audioData = frame->data[0];

 int audioDataSize = frame->linesize[0];

 if (s_pcm_callback) {

 s_pcm_callback(ffp->inject_opaque, pts, frame->format, frame->nb_samples, frame->channels, frame->sample_rate, frame->channel_layout, 1, size, targetData[0]);

 //                s_pcm_callback(audioData, audioDataSize, ffp_get_current_position_l(ffp)*1000);

 }

 free(targetData[0]);

 ffp_pcm_unlock(ffp);

 }

在解码前对音频做转格式初始化:

AVCodecContext *context = NULL;

 for (int i = 0; i < is->ic->nb_streams; i++) {

 // 对照输入流创建输出流通道

 AVStream *in_stream = is->ic->streams[i];

 if (in_stream->codec->codec_type == AVMEDIA_TYPE_AUDIO) {

 context = in_stream->codec;

 }

 }

 audio_swr_resampling_audio_init(&is->swr_ctx, context);

32位float型音频转位16位Int型音频代码:

void audio_swr_resampling_audio_destory(SwrContext **swr_ctx){

 if(*swr_ctx){

 swr_free(swr_ctx);

 *swr_ctx = NULL;

 }

}

void audio_swr_resampling_audio_init(SwrContext **swr_ctx,AVCodecContext *codec){

 if(codec->sample_fmt == AV_SAMPLE_FMT_S16 || codec->sample_fmt == AV_SAMPLE_FMT_S32 ||codec->sample_fmt == AV_SAMPLE_FMT_U8){

 av_log(NULL, AV_LOG_ERROR, "codec->sample_fmt:%d",codec->sample_fmt);

 if(*swr_ctx){

 swr_free(swr_ctx);

 *swr_ctx = NULL;

 }

 return;

 }

 if(*swr_ctx){

 swr_free(swr_ctx);

 }

 *swr_ctx = swr_alloc();

 if(!*swr_ctx){

 av_log(NULL, AV_LOG_ERROR, "swr_alloc failed");

 return;

 }

 /* set options */

 av_opt_set_int(*swr_ctx, "in_channel_layout",    codec->channel_layout, 0);

 av_opt_set_int(*swr_ctx, "out_channel_layout",    codec->channel_layout, 0);

 av_opt_set_int(*swr_ctx, "in_sample_rate",       codec->sample_rate, 0);

 av_opt_set_int(*swr_ctx, "out_sample_rate",       codec->sample_rate, 0);

 av_opt_set_sample_fmt(*swr_ctx, "in_sample_fmt", codec->sample_fmt, 0);

 av_opt_set_sample_fmt(*swr_ctx, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0);// AV_SAMPLE_FMT_S16

 /* initialize the resampling context */

 int ret = 0;

 if ((ret = swr_init(*swr_ctx)) < 0) {

 av_log(NULL, AV_LOG_ERROR, "Failed to initialize the resampling context\n");

 if(*swr_ctx){

 swr_free(swr_ctx);

 *swr_ctx = NULL;

 }

 return;

 }

}

int audio_swr_resampling_audio(SwrContext *swr_ctx,AVCodecContext *codec,AVFrame *audioFrame,uint8_t **targetData){

 uint8_t **extendedData = audioFrame->data;

 int len = swr_convert(swr_ctx,targetData,audioFrame->nb_samples,extendedData,audioFrame->nb_samples);

 if(len < 0){

 av_log(NULL, AV_LOG_ERROR, "error swr_convert");

 goto end;

 }

 int dst_bufsize = len * codec->channels * av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);

//    av_log(NULL, AV_LOG_INFO, " dst_bufsize:%d",dst_bufsize);

 return dst_bufsize;

 end:

 return -1;

}

至此,音视频原始数据已经回调到上层了

四、逐帧编码及写入

视频回调到上层已经是CVPixelBufferRef格式了,无需另外操作,直接使用AVAssetWriter写入到文件就行

CMTime tm = CMTimeMake(pts, 1000 * 1000);

BOOL success = [self.pixelBuffAdptor appendPixelBuffer:pixelBuff withPresentationTime:tm];

NSLog(@"___%d", success);

但是音频回调到上层是uint8_t *, 需要多一次音频数据处理变为iOS中适用的CMSampleBufferRef, 代码如下:

IJKWeakHolder *weakHolder = (__bridge IJKWeakHolder*)opaque;

 TCLIJKPlayer *mpc = weakHolder.object;

 if (!mpc) {

 return 0;

 }

 uint8_t *targetData = malloc(data_lineSize + mpc->lastCount);

 memcpy(targetData, mpc->lastData, mpc->lastCount);

 memcpy(targetData + mpc->lastCount, data, data_lineSize);

 int len = 2048;

 if(data_lineSize + mpc->lastCount < len) {

 memcpy(mpc->lastData + mpc->lastCount, data, data_lineSize);

 mpc->lastCount = data_lineSize + mpc->lastCount;

 free(targetData);

 return 0;

 }

 for (int i = 0; i <= (data_lineSize + mpc->lastCount)/len; i++) {

 if ((i+1)*len > (data_lineSize + mpc->lastCount)) {

 mpc->lastCount = (data_lineSize + mpc->lastCount) - i*len;

 memcpy(mpc->lastData, targetData + i*len, mpc->lastCount);

 } else {

 uint8_t *dst = malloc(len);

 memcpy(dst, targetData + i*len, len);

 CMSampleBufferRef buffer = createAudioSample(dst, len, pts, channels, sample_rate);

 free(dst);

 if (mpc.delegate && [mpc.delegate respondsToSelector:@selector(onAudioSampleBuffer:)]) {

 id buffRef = (__bridge id _Nullable)buffer;

 dispatch_async(dispatch_get_main_queue(), ^{

 [mpc.delegate onAudioSampleBuffer:(__bridge CMSampleBufferRef)(buffRef)];

 });

 }

 CFRelease(buffer);

 }

 }

 free(targetData);

static CMSampleBufferRef createAudioSample(void *audioData, UInt32 len, double pts, int channels, int sample_rate)

{

    int mDataByteSize = len;

    AudioBufferList audioBufferList;

    audioBufferList.mNumberBuffers = 1;

    audioBufferList.mBuffers[0].mNumberChannels=channels;

    audioBufferList.mBuffers[0].mDataByteSize=mDataByteSize;

    audioBufferList.mBuffers[0].mData = audioData;

    AudioStreamBasicDescription asbd;

    asbd.mSampleRate = sample_rate;

    asbd.mFormatID = kAudioFormatLinearPCM;

    asbd.mFormatFlags = kLinearPCMFormatFlagIsPacked | kLinearPCMFormatFlagIsSignedInteger;

    asbd.mChannelsPerFrame = channels;

    asbd.mBitsPerChannel = 16;

    asbd.mFramesPerPacket = 1;

    asbd.mBytesPerFrame = asbd.mBitsPerChannel / 8 * asbd.mChannelsPerFrame;

    asbd.mBytesPerPacket = asbd.mBytesPerFrame * asbd.mFramesPerPacket;

    asbd.mReserved = 0;

    static CMFormatDescriptionRef format = NULL;

    CMSampleTimingInfo timing = {CMTimeMake(1, sample_rate), kCMTimeZero, kCMTimeInvalid };

    OSStatus error = 0;

    error = CMAudioFormatDescriptionCreate(kCFAllocatorDefault, &asbd, 0, NULL, 0, NULL, NULL, &format);

    CMSampleBufferRef buff = NULL;

    error = CMSampleBufferCreate(kCFAllocatorDefault, NULL, false, NULL, NULL, format, (CMItemCount)mDataByteSize/(2*channels), 1, &timing, 0, NULL, &buff);

    CFRelease(format);

    if ( error ) {

        NSLog(@"CMSampleBufferCreate returned error: %ld", (long)error);

        return NULL;

    }

    error = CMSampleBufferSetDataBufferFromAudioBufferList(buff, kCFAllocatorDefault, kCFAllocatorDefault, 0, &audioBufferList);

    if( error )

    {

        NSLog(@"CMSampleBufferSetDataBufferFromAudioBufferList returned error: %ld", (long)error);

        return NULL;

    }

    return buff;

}

再使用AVAssetWriter写入到mp4

BOOL success = [self.assetWriterAudioInput appendSampleBuffer:sampleBuffer];

NSLog(@"-------%d", success);

if (!success) {

    @synchronized(self) {

        [self stopWrite:nil];

        [self destroyWrite];

    }

}

上一篇 下一篇

猜你喜欢

热点阅读