SurfaceTexture详解
2020-09-02 本文已影响0人
ben大福
之前讲到了flutter的Texture
SurfaceTexture 是 Surface 和 OpenGL ES (GLES) 纹理的组合。SurfaceTexture 用于提供输出到 GLES 纹理的 Surface
SurfaceTexture 包含一个 BufferQueue。当生产方将新的缓冲区排入队列时，onFrameAvailable() 回调会通知应用。然后，应用调用 updateTexImage()，这会释放先前占有的缓冲区，从队列中获取新缓冲区并执行 EGL 调用，从而使 GLES 可将此缓冲区作为外部纹理使用。
image.png
关键方法：
SurfaceTexture(int texName, boolean singleBufferMode)构造方法
setOnFrameAvailableListener 设置回调，当生产者准备好新的帧后会调用Listener
updateTexImage 更新texture到指定的GLESContext
detachFromGLContext
attachToGLContext
解绑/绑定 当前GLContext
getTransformMatrix 设置重采样纹理矩阵，当渲染的时候会用到这个数据
release() 完全释放 SufaceTexture的 buffers并且吧Surface状态置为abandoned
android-8.0.0_r1 源码解析：
public SurfaceTexture(int texName, boolean singleBufferMode) {
mCreatorLooper = Looper.myLooper();
mIsSingleBuffered = singleBufferMode;
nativeInit(false, texName, singleBufferMode, new WeakReference<SurfaceTexture>(this));
}
public SurfaceTexture(boolean singleBufferMode) {
mCreatorLooper = Looper.myLooper();
mIsSingleBuffered = singleBufferMode;
nativeInit(true, 0, singleBufferMode, new WeakReference<SurfaceTexture>(this));
}
static void SurfaceTexture_init(JNIEnv* env, jobject thiz, jboolean isDetached,
jint texName, jboolean singleBufferMode, jobject weakThiz){
sp<IGraphicBufferProducer> producer;
sp<IGraphicBufferConsumer> consumer;
BufferQueue::createBufferQueue(&producer, &consumer); //创建BufferQueue用于构建生产者消费者模式
if (singleBufferMode) {
consumer->setMaxBufferCount(1);
}
//创建GLConsumer消费者
sp<GLConsumer> surfaceTexture;
if (isDetached) {
surfaceTexture = new GLConsumer(consumer, GL_TEXTURE_EXTERNAL_OES,
true, !singleBufferMode);
} else {
surfaceTexture = new GLConsumer(consumer, texName,
GL_TEXTURE_EXTERNAL_OES, true, !singleBufferMode);
}
//无法申请texture报错
if (surfaceTexture == 0) {
jniThrowException(env, OutOfResourcesException,
"Unable to create native SurfaceTexture");
return;
}
surfaceTexture->setName(String8::format("SurfaceTexture-%d-%d-%d",
(isDetached ? 0 : texName),
getpid(),
createProcessUniqueId()));
// If the current context is protected, inform the producer.
consumer->setConsumerIsProtected(isProtectedContext());
// 将surfaceTexture保存到env中
SurfaceTexture_setSurfaceTexture(env, thiz, surfaceTexture);
// 将producer保存到env中
SurfaceTexture_setProducer(env, thiz, producer);
jclass clazz = env->GetObjectClass(thiz);
if (clazz == NULL) {
jniThrowRuntimeException(env,
"Can't find android/graphics/SurfaceTexture");
return;
}
sp<JNISurfaceTextureContext> ctx(new JNISurfaceTextureContext(env, weakThiz,
clazz));
surfaceTexture->setFrameAvailableListener(ctx);
SurfaceTexture_setFrameAvailableListener(env, thiz, ctx);
}
GLConsumer
GLConsumer(const sp<IGraphicBufferConsumer>& bq,
uint32_t tex, uint32_t texureTarget, bool useFenceSync,
bool isControlledByApp);
GLConsumer(const sp<IGraphicBufferConsumer>& bq, uint32_t texureTarget,
bool useFenceSync, bool isControlledByApp);
GLConsumer参数解释：
bq是BufferQueue创建BufferConsumer
tex 表示要将图像流传输到的OpenGL ES纹理名称。
texTarget指定了哪个纹理将被绑定
useFenceSync表示是否需要同步访问缓冲区
可以从一个OpenGL ES上下文中分离GLConsumer，然后分别使用detachFromContext和attachToContext方法将GLConsumer附加到另一个上下文。
如果设置tex参数则会通过attachToContext将GLConsumer附加到OpenGL ES context中。
第一次调用updateTexImage才会绑定，之后所有对updateTexImage的调用必须使用相同的当前OpenGL ES context进行
SurfaceTexture_updateTexImage
sp<GLConsumer> surfaceTexture(SurfaceTexture_getSurfaceTexture(env, thiz));
status_t err = surfaceTexture->updateTexImage();
if (err == INVALID_OPERATION) {
jniThrowException(env, IllegalStateException, "Unable to update texture contents (see "
"logcat for details)");
} else if (err < 0) {
jniThrowRuntimeException(env, "Error during updateTexImage (see logcat for details)");
}
GLConsumer.cpp
status_t GLConsumer::updateTexImage() {
······
status_t err = checkAndUpdateEglStateLocked();
BufferItem item;
// 获取下一个缓冲区
err = acquireBufferLocked(&item, 0);
if (err != NO_ERROR) {
if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
// We always bind the texture even if we don't update its contents.
GLC_LOGV("updateTexImage: no buffers were available");
glBindTexture(mTexTarget, mTexName);
err = NO_ERROR;
} else {
GLC_LOGE("updateTexImage: acquire failed: %s (%d)",
strerror(-err), err);
}
return err;
}
// 释放前一个缓冲区
err = updateAndReleaseLocked(item);
if (err != NO_ERROR) {
// We always bind the texture.
glBindTexture(mTexTarget, mTexName);
return err;
}
// 绑定新到的缓冲到纹理
return bindTextureImageLocked();
}
status_t GLConsumer::acquireBufferLocked(BufferItem *item,
nsecs_t presentWhen, uint64_t maxFrameNumber) {
status_t err = ConsumerBase::acquireBufferLocked(item, presentWhen,
maxFrameNumber);
if (err != NO_ERROR) {
return err;
}
// If item->mGraphicBuffer is not null, this buffer has not been acquired
// before, so any prior EglImage created is using a stale buffer. This
// replaces any old EglImage with a new one (using the new buffer).
if (item->mGraphicBuffer != NULL) {
int slot = item->mSlot;
mEglSlots[slot].mEglImage = new EglImage(item->mGraphicBuffer);
}
return NO_ERROR;
}
acquireBufferLocked创建EglImage并设置到EglSlots中
status_t GLConsumer::updateAndReleaseLocked(const BufferItem& item,
PendingRelease* pendingRelease)
{
······
int slot = item.mSlot;
·······
// Confirm state.
err = checkAndUpdateEglStateLocked();
err = mEglSlots[slot].mEglImage->createIfNeeded(mEglDisplay, item.mCrop);
// Do whatever sync ops we need to do before releasing the old slot.
if (slot != mCurrentTexture) {
err = syncForReleaseLocked(mEglDisplay);
if (err != NO_ERROR) {
// Release the buffer we just acquired. It's not safe to
// release the old buffer, so instead we just drop the new frame.
// As we are still under lock since acquireBuffer, it is safe to
// release by slot.
releaseBufferLocked(slot, mSlots[slot].mGraphicBuffer,
mEglDisplay, EGL_NO_SYNC_KHR);
return err;
}
}
// Hang onto the pointer so that it isn't freed in the call to
// releaseBufferLocked() if we're in shared buffer mode and both buffers are
// the same.
sp<EglImage> nextTextureImage = mEglSlots[slot].mEglImage;
// release old buffer
if (mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) {
if (pendingRelease == nullptr) {
status_t status = releaseBufferLocked(
mCurrentTexture, mCurrentTextureImage->graphicBuffer(),
mEglDisplay, mEglSlots[mCurrentTexture].mEglFence);
if (status < NO_ERROR) {
GLC_LOGE("updateAndRelease: failed to release buffer: %s (%d)",
strerror(-status), status);
err = status;
// keep going, with error raised [?]
}
} else {
pendingRelease->currentTexture = mCurrentTexture;
pendingRelease->graphicBuffer =
mCurrentTextureImage->graphicBuffer();
pendingRelease->display = mEglDisplay;
pendingRelease->fence = mEglSlots[mCurrentTexture].mEglFence;
pendingRelease->isPending = true;
}
}
// Update the GLConsumer state.
mCurrentTexture = slot;
mCurrentTextureImage = nextTextureImage;
mCurrentCrop = item.mCrop;
mCurrentTransform = item.mTransform;
mCurrentScalingMode = item.mScalingMode;
mCurrentTimestamp = item.mTimestamp;
mCurrentDataSpace = item.mDataSpace;
mCurrentFence = item.mFence;
mCurrentFenceTime = item.mFenceTime;
mCurrentFrameNumber = item.mFrameNumber;
computeCurrentTransformMatrixLocked();
return err;
}
status_t GLConsumer::EglImage::createIfNeeded(EGLDisplay eglDisplay,
const Rect& cropRect,
bool forceCreation) {
// If there's an image and it's no longer valid, destroy it.
bool haveImage = mEglImage != EGL_NO_IMAGE_KHR;
bool displayInvalid = mEglDisplay != eglDisplay;
bool cropInvalid = hasEglAndroidImageCrop() && mCropRect != cropRect;
if (haveImage && (displayInvalid || cropInvalid || forceCreation)) {
if (!eglDestroyImageKHR(mEglDisplay, mEglImage)) {
ALOGE("createIfNeeded: eglDestroyImageKHR failed");
}
eglTerminate(mEglDisplay);
mEglImage = EGL_NO_IMAGE_KHR;
mEglDisplay = EGL_NO_DISPLAY;
}
// If there's no image, create one.
if (mEglImage == EGL_NO_IMAGE_KHR) {
mEglDisplay = eglDisplay;
mCropRect = cropRect;
mEglImage = createImage(mEglDisplay, mGraphicBuffer, mCropRect);
}
······
return OK;
}
updateAndReleaseLocked 更新 EglImage
createIfNeeded 如果EGLDisplay改变或者crop改变则会创建EglImage
bindToTextureTarget 将调用glEGLImageTargetTexture2DOES去绑定image到指定的目标纹理
EglImage
status_t GLConsumer::EglImage::createIfNeeded(EGLDisplay eglDisplay,
const Rect& cropRect,
bool forceCreation) {
bool haveImage = mEglImage != EGL_NO_IMAGE_KHR;
bool displayInvalid = mEglDisplay != eglDisplay;
bool cropInvalid = hasEglAndroidImageCrop() && mCropRect != cropRect;
//如果EGLDisplay改变或者crop改变则会创建EglImage，则销毁之前的EglImage
if (haveImage && (displayInvalid || cropInvalid || forceCreation)) {
if (!eglDestroyImageKHR(mEglDisplay, mEglImage)) {
ALOGE("createIfNeeded: eglDestroyImageKHR failed");
}
eglTerminate(mEglDisplay);
mEglImage = EGL_NO_IMAGE_KHR;
mEglDisplay = EGL_NO_DISPLAY;
}
// 创建新的EglImage
if (mEglImage == EGL_NO_IMAGE_KHR) {
mEglDisplay = eglDisplay;
mCropRect = cropRect;
mEglImage = createImage(mEglDisplay, mGraphicBuffer, mCropRect);
}
·······
}
void GLConsumer::EglImage::bindToTextureTarget(uint32_t texTarget) {
glEGLImageTargetTexture2DOES(texTarget,
static_cast<GLeglImageOES>(mEglImage));
}
EGLImageKHR GLConsumer::EglImage::createImage(EGLDisplay dpy,
const sp<GraphicBuffer>& graphicBuffer, const Rect& crop) {
EGLClientBuffer cbuf =
static_cast<EGLClientBuffer>(graphicBuffer->getNativeBuffer());
const bool createProtectedImage =
(graphicBuffer->getUsage() & GRALLOC_USAGE_PROTECTED) &&
hasEglProtectedContent();
EGLint attrs[] = {
EGL_IMAGE_PRESERVED_KHR, EGL_TRUE,
EGL_IMAGE_CROP_LEFT_ANDROID, crop.left,
EGL_IMAGE_CROP_TOP_ANDROID, crop.top,
EGL_IMAGE_CROP_RIGHT_ANDROID, crop.right,
EGL_IMAGE_CROP_BOTTOM_ANDROID, crop.bottom,
createProtectedImage ? EGL_PROTECTED_CONTENT_EXT : EGL_NONE,
createProtectedImage ? EGL_TRUE : EGL_NONE,
EGL_NONE,
};
if (!crop.isValid()) {
// No crop rect to set, so leave the crop out of the attrib array. Make
// sure to propagate the protected content attrs if they are set.
attrs[2] = attrs[10];
attrs[3] = attrs[11];
attrs[4] = EGL_NONE;
} else if (!isEglImageCroppable(crop)) {
// The crop rect is not at the origin, so we can't set the crop on the
// EGLImage because that's not allowed by the EGL_ANDROID_image_crop
// extension. In the future we can add a layered extension that
// removes this restriction if there is hardware that can support it.
attrs[2] = attrs[10];
attrs[3] = attrs[11];
attrs[4] = EGL_NONE;
}
eglInitialize(dpy, 0, 0);
EGLImageKHR image = eglCreateImageKHR(dpy, EGL_NO_CONTEXT,
EGL_NATIVE_BUFFER_ANDROID, cbuf, attrs);
if (image == EGL_NO_IMAGE_KHR) {
EGLint error = eglGetError();
ALOGE("error creating EGLImage: %#x", error);
eglTerminate(dpy);
}
return image;
}
这里创建EGLImageKHR，EGLImageKHR用于共享EGL资源
EGL的ShareContext
EGL的ShareContext是常见的共享上下文的方式（iOS平台的EAGL叫ShareGroup）。
/**
share_context:
Specifies another EGL rendering context with which to share data, as defined by the client API corresponding to the contexts. Data is also shared with all other contexts with which share_context shares data. EGL_NO_CONTEXT indicates that no sharing is to take place.
**/
EGLContext eglCreateContext( EGLDisplay display,
EGLConfig config,
EGLContext share_context,
EGLint const * attrib_list)
当share_context参数传入另一个EGL的context时，这两个EGLContext就可以共享纹理以及VBO等。
需要注意的是container objects不能被共享，比如：
Framebuffer objects
Vertex array objects
Transform feedback objects
Program pipeline objects
参考：https://wiki.jikexueyuan.com/project/deep-android-v1/classes.html
EGLImageKHR:https://www.khronos.org/registry/EGL/extensions/KHR/EGL_KHR_image_base.txt
