openGL ES进阶教程（五）制作一个简单的VR播放器，播放全
2017-08-02 本文已影响192人
小码哥_WS
之前写过全景（VR）图片，和用openGL ES+MediaPlayer 渲染播放视频+滤镜效果
这一篇就在之前的基础上实现一个最简单的VR播放器，播放全景视频。
概述：
全景视频是一种用3D摄像机进行全方位360度进行拍摄的视频，用户在观看视频的时候，可以随意调节视频上下左右进行观看。
展示：
“身临其境的枪战” 可见晃动手机，视图也跟着转！
这里写图片描述
原理：
1.用OpenGL绘制一个球。
2.MediaPlayer 播放的视频纹理贴到球上（可以想象为一个地球仪）。
3.把观测点设置在求内部。 （想象为从球内部中心点观看球的表面）。
4.用手机内置的传感器得到一个手机移动的矩阵（视频画面就可以随着手机的移动而移动了）。
实践：
1.用OpenGL绘制一个球
效果如图：
这里写图片描述
用OpenGL绘制任何图形，都需要先知道顶点坐标。然后用GLES20.glDrawArrays绘制出来。
那么绘制一个球体，可以想象为无数个三角形在三维空间构成。
下面这个算法就会放回一个球体的坐标数组
private float[] createBallPos(){
//球以(0,0,0)为中心，以R为半径，则球上任意一点的坐标为
// ( R * cos(a) * sin(b),y0 = R * sin(a),R * cos(a) * cos(b))
// 其中，a为圆心到点的线段与xz平面的夹角，b为圆心到点的线段在xz平面的投影与z轴的夹角
ArrayList<Float> data=new ArrayList<>();
float r1,r2;
float h1,h2;
float sin,cos;
for(float i=-90;i<90+step;i+=step){
r1 = (float)Math.cos(i * Math.PI / 180.0);
r2 = (float)Math.cos((i + step) * Math.PI / 180.0);
h1 = (float)Math.sin(i * Math.PI / 180.0);
h2 = (float)Math.sin((i + step) * Math.PI / 180.0);
// 固定纬度, 360 度旋转遍历一条纬线
float step2=step*2;
for (float j = 0.0f; j <360.0f+step; j +=step2 ) {
cos = (float) Math.cos(j * Math.PI / 180.0);
sin = -(float) Math.sin(j * Math.PI / 180.0);
data.add(r2 * cos);
data.add(h2);
data.add(r2 * sin);
data.add(r1 * cos);
data.add(h1);
data.add(r1 * sin);
}
}
float[] f=new float[data.size()];
for(int i=0;i<f.length;i++){
f[i]=data.get(i);
}
return f;
}
然后把坐标传给natave层
float[] dataPos=createBallPos();
vertexBuffer = ByteBuffer.allocateDirect(dataPos.length * 4)
.order(ByteOrder.nativeOrder())
.asFloatBuffer()
.put(dataPos);
vertexBuffer.position(0);
vSize=dataPos.length/3;
有了坐标就可以绘制球了。
2.MediaPlayer 播放的视频纹理贴到球上（可以想象为一个地球仪）。
先贴着色器代码
attribute vec4 aPosition;//顶点位置
attribute vec4 aTexCoord;//S T 纹理坐标
varying vec2 vTexCoord;
uniform mat4 uMatrix;
uniform mat4 uSTMatrix;
uniform mat4 uViewMatrix;
uniform mat4 uModelMatrix;
uniform mat4 uRotateMatrix;
void main() {
vTexCoord = (uSTMatrix * aTexCoord).xy;
gl_Position = uMatrix*uRotateMatrix*uViewMatrix*uModelMatrix*aPosition;
}
#extension GL_OES_EGL_image_external : require
precision mediump float;
varying vec2 vTexCoord;
uniform samplerExternalOES sTexture;
void main() {
gl_FragColor=texture2D(sTexture, vTexCoord);
}
然后我们根据球的点把视频纹理贴到球上
GLES20.glEnableVertexAttribArray(aPositionLocation);
GLES20.glVertexAttribPointer(aPositionLocation, 3, GLES20.GL_FLOAT, false,
0, posBuffer);
GLES20.glEnableVertexAttribArray(aTextureCoordLocation);
GLES20.glVertexAttribPointer(aTextureCoordLocation,2,GLES20.GL_FLOAT,false,0,cooBuffer);
绘制：
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, vCount);
3.把观测点设置在求内部。 （想象为从球内部中心点观看球的表面）。
//设置相机位置
Matrix.setLookAtM(mViewMatrix, 0, 0f, 0.0f,0.0f, 0.0f, 0.0f,-1.0f, 0f,-1.0f, 0.0f);
4.用手机内置的传感器得到一个手机移动的矩阵（视频画面就可以随着手机的移动而移动了）
在activity中
@Override
public void onSensorChanged(SensorEvent sensorEvent) {
SensorManager.getRotationMatrixFromVector(matrix,sensorEvent.values);
glRenderer.setMatrix(matrix);
}
VRVideoRenderer 完整代码如下：
import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.AudioManager;
import android.media.MediaPlayer;
import android.net.Uri;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.util.ArrayList;
import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;
/**
* Created by Shuo.Wang on 2017/4/19.
*/
public class VRVideoRenderer implements GLSurfaceView.Renderer
, SurfaceTexture.OnFrameAvailableListener, MediaPlayer.OnVideoSizeChangedListener {
private static final String TAG = "GLRenderer";
private static final float UNIT_SIZE = 1f;// 单位尺寸
private float radius=2f;
final double angleSpan = Math.PI/90f;// 将球进行单位切分的角度
int vCount = 0;// 顶点个数，先初始化为0
private FloatBuffer posBuffer;
private FloatBuffer cooBuffer;
private int mHViewMatrix;
private int mHModelMatrix;
private int mHRotateMatrix;
private float[] mViewMatrix=new float[16];
private float[] mModelMatrix=new float[16];
private float[] mRotateMatrix=new float[16];
private Context context;
private int aPositionLocation;
private int programId;
private FloatBuffer vertexBuffer;
private final float[] projectionMatrix=new float[16];
private int uMatrixLocation;
private FloatBuffer textureVertexBuffer;
private int uTextureSamplerLocation;
private int aTextureCoordLocation;
private int textureId;
private SurfaceTexture surfaceTexture;
private MediaPlayer mediaPlayer;
private float[] mSTMatrix = new float[16];
private int uSTMMatrixHandle;
private boolean updateSurface;
private boolean playerPrepared;
private int screenWidth,screenHeight;
public VRVideoRenderer(Context context, String videoPath) {
this.context = context;
playerPrepared=false;
synchronized(this) {
updateSurface = false;
}
calculateAttribute();
mediaPlayer=new MediaPlayer();
try{
mediaPlayer.setDataSource(context, Uri.parse(videoPath));
}catch (IOException e){
e.printStackTrace();
}
mediaPlayer.setAudioStreamType(AudioManager.STREAM_MUSIC);
mediaPlayer.setLooping(true);
mediaPlayer.setOnVideoSizeChangedListener(this);
}
@Override
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
String vertexShader = ShaderUtils.readRawTextFile(context, R.raw.simple_vertex_shader);
String fragmentShader= ShaderUtils.readRawTextFile(context, R.raw.simple_fragment_shader);
programId=ShaderUtils.createProgram(vertexShader,fragmentShader);
aPositionLocation= GLES20.glGetAttribLocation(programId,"aPosition");
uMatrixLocation=GLES20.glGetUniformLocation(programId,"uMatrix");
uSTMMatrixHandle = GLES20.glGetUniformLocation(programId, "uSTMatrix");
mHViewMatrix=GLES20.glGetUniformLocation(programId,"uViewMatrix");
mHModelMatrix=GLES20.glGetUniformLocation(programId,"uModelMatrix");
mHRotateMatrix=GLES20.glGetUniformLocation(programId,"uRotateMatrix");
uTextureSamplerLocation=GLES20.glGetUniformLocation(programId,"sTexture");
aTextureCoordLocation=GLES20.glGetAttribLocation(programId,"aTexCoord");
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
textureId = textures[0];
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);
ShaderUtils.checkGlError("glBindTexture mTextureID");
/*GLES11Ext.GL_TEXTURE_EXTERNAL_OES的用处？
之前提到视频解码的输出格式是YUV的（YUV420p，应该是），那么这个扩展纹理的作用就是实现YUV格式到RGB的自动转化，
我们就不需要再为此写YUV转RGB的代码了*/
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
GLES20.GL_LINEAR);
surfaceTexture = new SurfaceTexture(textureId);
surfaceTexture.setOnFrameAvailableListener(this);//监听是否有新的一帧数据到来
Surface surface = new Surface(surfaceTexture);
mediaPlayer.setSurface(surface);
surface.release();
if (!playerPrepared){
try {
mediaPlayer.prepare();
playerPrepared=true;
} catch (IOException t) {
Log.e(TAG, "media player prepare failed");
}
mediaPlayer.start();
playerPrepared=true;
}
}
@Override
public void onSurfaceChanged(GL10 gl, int width, int height) {
Log.d(TAG, "onSurfaceChanged: "+width+" "+height);
screenWidth=width; screenHeight=height;
//计算宽高比
float ratio=(float)width/height;
//透视投影矩阵/视锥
MatrixHelper.perspectiveM(projectionMatrix,0,90,ratio,1f,500);
//设置相机位置
Matrix.setLookAtM(mViewMatrix, 0, 0f, 0.0f,0.0f, 0.0f, 0.0f,-1.0f, 0f,-1.0f, 0.0f);
//模型矩阵
Matrix.setIdentityM(mModelMatrix,0);
Matrix.rotateM(mModelMatrix,0,180f,1f,0f,0f);
}
@Override
public void onDrawFrame(GL10 gl) {
GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);
synchronized (this){
if (updateSurface){
surfaceTexture.updateTexImage();//获取新数据
surfaceTexture.getTransformMatrix(mSTMatrix);//让新的纹理和纹理坐标系能够正确的对应,mSTMatrix的定义是和projectionMatrix完全一样的。
updateSurface = false;
}
}
GLES20.glUseProgram(programId);
GLES20.glUniformMatrix4fv(uMatrixLocation,1,false,projectionMatrix,0);
GLES20.glUniformMatrix4fv(uSTMMatrixHandle, 1, false, mSTMatrix, 0);
GLES20.glUniformMatrix4fv(mHViewMatrix,1,false,mViewMatrix,0);
GLES20.glUniformMatrix4fv(mHModelMatrix,1,false,mModelMatrix,0);
GLES20.glUniformMatrix4fv(mHRotateMatrix,1,false,mRotateMatrix,0);
GLES20.glEnableVertexAttribArray(aPositionLocation);
GLES20.glVertexAttribPointer(aPositionLocation, 3, GLES20.GL_FLOAT, false,
0, posBuffer);
GLES20.glEnableVertexAttribArray(aTextureCoordLocation);
GLES20.glVertexAttribPointer(aTextureCoordLocation,2,GLES20.GL_FLOAT,false,0,cooBuffer);
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,textureId);
GLES20.glUniform1i(uTextureSamplerLocation,0);
GLES20.glViewport(0,0,screenWidth,screenHeight);
//GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
GLES20.glDrawArrays(GLES20.GL_TRIANGLES, 0, vCount);//added by wangshuo
}
@Override
synchronized public void onFrameAvailable(SurfaceTexture surface) {
updateSurface = true;
}
@Override
public void onVideoSizeChanged(MediaPlayer mp, int width, int height) {
Log.d(TAG, "onVideoSizeChanged: "+width+" "+height);
//updateProjection(width,height);
}
public MediaPlayer getMediaPlayer() {
return mediaPlayer;
}
public void setMatrix(float[] matrix){
System.arraycopy(matrix,0,mRotateMatrix,0,16);
}
private void calculateAttribute(){
ArrayList<Float> alVertix = new ArrayList<>();
ArrayList<Float> textureVertix = new ArrayList<>();
for (double vAngle = 0; vAngle < Math.PI; vAngle = vAngle + angleSpan){
for (double hAngle = 0; hAngle < 2*Math.PI; hAngle = hAngle + angleSpan){
float x0 = (float) (radius* Math.sin(vAngle) * Math.cos(hAngle));
float y0 = (float) (radius* Math.sin(vAngle) * Math.sin(hAngle));
float z0 = (float) (radius * Math.cos((vAngle)));
float x1 = (float) (radius* Math.sin(vAngle) * Math.cos(hAngle + angleSpan));
float y1 = (float) (radius* Math.sin(vAngle) * Math.sin(hAngle + angleSpan));
float z1 = (float) (radius * Math.cos(vAngle));
float x2 = (float) (radius* Math.sin(vAngle + angleSpan) * Math.cos(hAngle + angleSpan));
float y2 = (float) (radius* Math.sin(vAngle + angleSpan) * Math.sin(hAngle + angleSpan));
float z2 = (float) (radius * Math.cos(vAngle + angleSpan));
float x3 = (float) (radius* Math.sin(vAngle + angleSpan) * Math.cos(hAngle));
float y3 = (float) (radius* Math.sin(vAngle + angleSpan) * Math.sin(hAngle));
float z3 = (float) (radius * Math.cos(vAngle + angleSpan));
alVertix.add(x1);
alVertix.add(y1);
alVertix.add(z1);
alVertix.add(x0);
alVertix.add(y0);
alVertix.add(z0);
alVertix.add(x3);
alVertix.add(y3);
alVertix.add(z3);
float s0 = (float) (hAngle / Math.PI/2);
float s1 = (float) ((hAngle + angleSpan)/Math.PI/2);
float t0 = (float) (vAngle / Math.PI);
float t1 = (float) ((vAngle + angleSpan) / Math.PI);
textureVertix.add(s1);// x1 y1对应纹理坐标
textureVertix.add(t0);
textureVertix.add(s0);// x0 y0对应纹理坐标
textureVertix.add(t0);
textureVertix.add(s0);// x3 y3对应纹理坐标
textureVertix.add(t1);
alVertix.add(x1);
alVertix.add(y1);
alVertix.add(z1);
alVertix.add(x3);
alVertix.add(y3);
alVertix.add(z3);
alVertix.add(x2);
alVertix.add(y2);
alVertix.add(z2);
textureVertix.add(s1);// x1 y1对应纹理坐标
textureVertix.add(t0);
textureVertix.add(s0);// x3 y3对应纹理坐标
textureVertix.add(t1);
textureVertix.add(s1);// x2 y3对应纹理坐标
textureVertix.add(t1);
}
}
vCount = alVertix.size() / 3;
posBuffer = convertToFloatBuffer(alVertix);
cooBuffer=convertToFloatBuffer(textureVertix);
}
private FloatBuffer convertToFloatBuffer(ArrayList<Float> data){
float[] d=new float[data.size()];
for (int i=0;i<d.length;i++){
d[i]=data.get(i);
}
ByteBuffer buffer=ByteBuffer.allocateDirect(data.size()*4);
buffer.order(ByteOrder.nativeOrder());
FloatBuffer ret=buffer.asFloatBuffer();
ret.put(d);
ret.position(0);
return ret;
}
}
