macOS 屏幕录制与窗口捕获

完整 demo MacScreenCapture

1. 屏幕录制方案，采用AVCaptureScreenInput
2. 窗口捕获，采用CoreGraphics.framework

屏幕录制

相对比较简单

• 使用 AVCaptureSession, 指定采集的输入为AVCaptureScreenInput，配置输入(是否采集鼠标，采集的帧率，裁剪矩形等)
• 指定输出为 AVCaptureVideoDataOutput
• AVCaptureSession startRunning

//AVCaptureScreenInput.h
- (nullable instancetype)initWithDisplayID:(CGDirectDisplayID)displayID;

获取 displayID

for (NSScreen *screen in NSScreen.screens) {
    NSNumber *number = [screen.deviceDescription objectForKey:@"NSScreenNumber"];
    NSLog(@"%@",number);
}

一些参数的使用

//The origin (0,0) is the bottom-left corner of the screen.
@property(nonatomic) CGRect cropRect;

//指定帧率
@property(nonatomic) CMTime minFrameDuration;

//是否采集鼠标
@property(nonatomic) BOOL capturesCursor API_AVAILABLE(macos(10.8));

窗口捕获

1. 指定想要获取的窗口 id
2. 通过窗口 id，设置定时器，定时将窗口捕获为图片（RGB格式）
3. 获取鼠标的位置，将鼠标和窗口图像进行合成
4. 将图片根据裁剪矩形进行裁剪
5. 最终的图片，转换为 nv12 或者 i420

获取窗口的 id， 参考 stackoverflow

苹果提供了一个 SonOfGrab，可以很方便获取所有的窗口 ID

CFArrayRef windowList = CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements, kCGNullWindowID);
CFIndex count = CFArrayGetCount(windowList);
if (count == 0) {
    return;
}

for (CFIndex i = 0; i < count; i++) {
    CFDictionaryRef window = CFArrayGetValueAtIndex(windowList, i);
    CFNumberRef widowID = CFDictionaryGetValue(window, kCGWindowNumber);
    NSLog(@"window ID = %@", widowID);
}

捕获窗口为图片

//取得窗口快照
CGImageRef windowImage = CGWindowListCreateImage(rect, kCGWindowListOptionIncludingWindow | kCGWindowListExcludeDesktopElements, (CGWindowID)self.windowID, kCGWindowImageNominalResolution);
if (!windowImage) {
    NSLog(@"window image is null");
    return;
}

追加鼠标，参考 https://www.coder.work/article/1297008

-(CGImageRef)appendMouseCursor:(CGImageRef)pSourceImage sourceImageRect:(CGRect)imageRect {
    // get the cursor image
    
    if (!pSourceImage) {
        return NULL;
    }
    
    //imageRect 坐标在左上角， 转换为左下角
    CGRect imageRect_BottomLeft = imageRect;
    CGFloat y = NSMaxY(NSScreen.mainScreen.frame) - CGRectGetMaxY(imageRect);
    imageRect_BottomLeft.origin.y = y;

    //坐标在左下角
    CGPoint mouseLoc = [NSEvent mouseLocation];
        
    // get the mouse image
    NSImage *overlay = [[NSCursor currentSystemCursor] image];
    
    CGImageRef overlayImage = [overlay CGImageForProposedRect:NULL
                                                      context:nil hints:nil];
    
    if (CGImageGetWidth(overlayImage) != (size_t)overlay.size.width) {
        NSLog(@"should scale");
    }
    
    CGRect mouseRect = CGRectMake(mouseLoc.x,  mouseLoc.y, overlay.size.width, overlay.size.height);
    
    if (!CGRectContainsRect(imageRect_BottomLeft, mouseRect)) {
        CFRetain(pSourceImage);
        return pSourceImage;
    }
    
    CGPoint convertedPoint = CGPointMake(mouseRect.origin.x - imageRect_BottomLeft.origin.x, mouseRect.origin.y - imageRect_BottomLeft.origin.y);

    CGRect cursorRect = CGRectMake(convertedPoint.x, convertedPoint.y, overlay.size.width, overlay.size.height);
        
    size_t height = CGImageGetHeight(pSourceImage);
    size_t width =  CGImageGetWidth(pSourceImage);
    int bytesPerRow = (int)CGImageGetBytesPerRow(pSourceImage);

    unsigned int * imgData = (unsigned int*)malloc(height*bytesPerRow);
    // have the graphics context now,
    CGRect bgBoundingBox = CGRectMake (0, 0, width,height);
    CGContextRef context =  CGBitmapContextCreate(imgData, width,
                                                  height,
                                                  8, // 8 bits per component
                                                  bytesPerRow,
                                                  CGImageGetColorSpace(pSourceImage),
                                                  CGImageGetBitmapInfo(pSourceImage));

    // first draw the image
    CGContextDrawImage(context,bgBoundingBox,pSourceImage);

    NSRect overlayRect = CGRectMake(0, 0, overlay.size.width, overlay.size.height);
    // then mouse cursor
    CGContextDrawImage(context, cursorRect, [overlay CGImageForProposedRect:&overlayRect context:NULL hints:NULL]);
    // assuming both the image has been drawn then create an Image Ref for that

    CGImageRef pFinalImage = CGBitmapContextCreateImage(context);

    CGContextRelease(context);
    free(imgData);

    return pFinalImage; /* to be released by the caller */
}

裁剪图片，注意图片的原点在左上角

//做裁切,
CGRect cropRect = CGRectInfinite;
//    cropRect = CGRectMake(0, 0, 640, 360);
CGImageRef croppedImage = CGImageCreateWithImageInRect(imagWithCursor, cropRect);
CFRelease(imagWithCursor);
if (!croppedImage) {
    return;
}

将图片转为 i420 pixelBuffer

- (CVPixelBufferRef)pixelBufferFromCGImage: (CGImageRef) image
{
    NSCParameterAssert(NULL != image);
    size_t originalWidth = CGImageGetWidth(image);
    size_t originalHeight = CGImageGetHeight(image);
    
    if (originalWidth == 0 || originalHeight == 0) {
        return NULL;
    }

    size_t bytePerRow = CGImageGetBytesPerRow(image);
    CFDataRef data  = CGDataProviderCopyData(CGImageGetDataProvider(image));
    const UInt8 *ptr =  CFDataGetBytePtr(data);
    
    //create rgb buffer
    NSDictionary *att = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{} };
    
    CVPixelBufferRef buffer;
    CVPixelBufferCreateWithBytes(kCFAllocatorDefault,
                                 originalWidth,
                                 originalHeight,
                                 kCVPixelFormatType_32BGRA,
                                 (void *)ptr,
                                 bytePerRow,
                                 _CVPixelBufferReleaseBytesCallback,
                                 (void *)data,
                                 (__bridge CFDictionaryRef _Nullable)att,
                                 &buffer);
    
    
    CVPixelBufferLockBaseAddress(buffer, 0);
    int width = CVPixelBufferGetWidth(buffer);
    int height = CVPixelBufferGetHeight(buffer);
        
    //防止出现绿边
    height = height - height%2;

    CVPixelBufferRef i420Buffer;
    CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_420YpCbCr8Planar, (__bridge CFDictionaryRef _Nullable)att,&i420Buffer);
    CVPixelBufferLockBaseAddress(i420Buffer, 0);
    
    void *y_frame = CVPixelBufferGetBaseAddressOfPlane(i420Buffer, 0);
    void *u_frame = CVPixelBufferGetBaseAddressOfPlane(i420Buffer, 1);
    void *v_frame = CVPixelBufferGetBaseAddressOfPlane(i420Buffer, 2);

    
    int stride_y = CVPixelBufferGetBytesPerRowOfPlane(i420Buffer, 0);
    int stride_u = CVPixelBufferGetBytesPerRowOfPlane(i420Buffer, 1);
    int stride_v = CVPixelBufferGetBytesPerRowOfPlane(i420Buffer, 2);
    
    
    void *rgb = CVPixelBufferGetBaseAddressOfPlane(buffer, 0);
    void *rgb_stride = CVPixelBufferGetBytesPerRow(buffer);
    
    
    ARGBToI420(rgb, rgb_stride,
               y_frame, stride_y,
               u_frame, stride_u,
               v_frame, stride_v,
               width, height);
    
    CVPixelBufferUnlockBaseAddress(i420Buffer, 0);
    CVPixelBufferUnlockBaseAddress(buffer, 0);
    CVPixelBufferRelease(buffer);
    
    return  i420Buffer;
}

void _CVPixelBufferReleaseBytesCallback(void *releaseRefCon, const void *baseAddress) {
    
    CFDataRef data = releaseRefCon;
    CFRelease(data);
    
}