From 55c617aa2d49e3403f8700b173dc9220654deb74 Mon Sep 17 00:00:00 2001 From: Eino-Ville Talvala Date: Fri, 4 Mar 2011 11:28:28 -0800 Subject: [PATCH] Add a simple OpenGL ES 2.0 test program that displays a camera preview stream. Uses a SurfaceTexture feeding into a OpenGL ES2 GLSurfaceView. Also uses accelerometer data and SurfaceTexture-provided timestamps for a simple physics simulation which moves the camera preview oval in response to device movement. Change-Id: I275a0c6a4e25829d617e635b25e70e2a9c8455cb --- opengl/tests/gl2_cameraeye/Android.mk | 16 + .../tests/gl2_cameraeye/AndroidManifest.xml | 36 ++ .../gl2_cameraeye/res/values/strings.xml | 21 + .../android/gl2cameraeye/GL2CameraEye.java | 508 ++++++++++++++++++ 4 files changed, 581 insertions(+) create mode 100644 opengl/tests/gl2_cameraeye/Android.mk create mode 100644 opengl/tests/gl2_cameraeye/AndroidManifest.xml create mode 100644 opengl/tests/gl2_cameraeye/res/values/strings.xml create mode 100644 opengl/tests/gl2_cameraeye/src/com/android/gl2cameraeye/GL2CameraEye.java diff --git a/opengl/tests/gl2_cameraeye/Android.mk b/opengl/tests/gl2_cameraeye/Android.mk new file mode 100644 index 000000000..4a43a9e2f --- /dev/null +++ b/opengl/tests/gl2_cameraeye/Android.mk @@ -0,0 +1,16 @@ +LOCAL_PATH:= $(call my-dir) +include $(CLEAR_VARS) + +LOCAL_MODULE_TAGS := optional + +# Only compile source java files in this apk. +LOCAL_SRC_FILES := $(call all-java-files-under, src) + +LOCAL_PACKAGE_NAME := GL2CameraEye + +LOCAL_SDK_VERSION := current + +include $(BUILD_PACKAGE) + +# Use the following include to make our test apk. +include $(call all-makefiles-under,$(LOCAL_PATH)) diff --git a/opengl/tests/gl2_cameraeye/AndroidManifest.xml b/opengl/tests/gl2_cameraeye/AndroidManifest.xml new file mode 100644 index 000000000..c53f7be0b --- /dev/null +++ b/opengl/tests/gl2_cameraeye/AndroidManifest.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + diff --git a/opengl/tests/gl2_cameraeye/res/values/strings.xml b/opengl/tests/gl2_cameraeye/res/values/strings.xml new file mode 100644 index 000000000..386b93030 --- /dev/null +++ b/opengl/tests/gl2_cameraeye/res/values/strings.xml @@ -0,0 +1,21 @@ + + + + + + GL2CameraEye + + diff --git a/opengl/tests/gl2_cameraeye/src/com/android/gl2cameraeye/GL2CameraEye.java b/opengl/tests/gl2_cameraeye/src/com/android/gl2cameraeye/GL2CameraEye.java new file mode 100644 index 000000000..561e4c50f --- /dev/null +++ b/opengl/tests/gl2_cameraeye/src/com/android/gl2cameraeye/GL2CameraEye.java @@ -0,0 +1,508 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.gl2cameraeye; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; + +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +import android.app.Activity; +import android.content.pm.ActivityInfo; +import android.os.Bundle; +import android.view.MotionEvent; +import android.content.Context; +import android.util.Log; + +import android.opengl.GLES20; +import android.opengl.GLSurfaceView; +import android.opengl.GLUtils; +import android.opengl.Matrix; + +import android.graphics.SurfaceTexture; + +import android.hardware.Camera; +import android.hardware.SensorManager; +import android.hardware.SensorEvent; +import android.hardware.SensorEventListener; +import android.hardware.Sensor; + +public class GL2CameraEye extends Activity { + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + mGLView = new CamGLSurfaceView(this); + setContentView(mGLView); + setRequestedOrientation (ActivityInfo.SCREEN_ORIENTATION_LANDSCAPE); + } + + @Override + protected void onPause() { + super.onPause(); + mGLView.onPause(); + } + + @Override + protected void onResume() { + super.onResume(); + mGLView.onResume(); + } + + private GLSurfaceView mGLView; +} + +class CamGLSurfaceView extends GLSurfaceView implements SensorEventListener { + public CamGLSurfaceView(Context context) { + super(context); + setEGLContextClientVersion(2); + mRenderer = new CamRenderer(context); + setRenderer(mRenderer); + + mSensorManager = (SensorManager)context.getSystemService(Context.SENSOR_SERVICE); + mAcceleration = mSensorManager.getDefaultSensor(Sensor.TYPE_LINEAR_ACCELERATION); + } + + public boolean onTouchEvent(final MotionEvent event) { + queueEvent(new Runnable(){ + public void run() { + mRenderer.setPosition(event.getX() / getWidth(), + event.getY() / getHeight()); + }}); + return true; + } + + @Override + public void onPause() { + super.onPause(); + mCamera.stopPreview(); + mCamera.release(); + + mSensorManager.unregisterListener(this); + } + + @Override + public void onResume() { + mCamera = Camera.open(); + Camera.Parameters p = mCamera.getParameters(); + // No changes to default camera parameters + mCamera.setParameters(p); + + queueEvent(new Runnable(){ + public void run() { + mRenderer.setCamera(mCamera); + }}); + + mSensorManager.registerListener(this, mAcceleration, SensorManager.SENSOR_DELAY_GAME); + super.onResume(); + } + + public void onSensorChanged(SensorEvent event) { + if (event.sensor.getType() == Sensor.TYPE_LINEAR_ACCELERATION) { + final float[] accelerationVector = event.values; + queueEvent(new Runnable(){ + public void run() { + mRenderer.setAcceleration(accelerationVector); + }}); + } + } + + public void onAccuracyChanged(Sensor sensor, int accuracy) { + // Ignoring sensor accuracy changes. + } + + CamRenderer mRenderer; + Camera mCamera; + + SensorManager mSensorManager; + Sensor mAcceleration; +} + +class CamRenderer implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener { + + public CamRenderer(Context context) { + mContext = context; + + mTriangleVertices = ByteBuffer.allocateDirect(mTriangleVerticesData.length + * FLOAT_SIZE_BYTES).order(ByteOrder.nativeOrder()).asFloatBuffer(); + mTriangleVertices.put(mTriangleVerticesData).position(0); + + Matrix.setIdentityM(mSTMatrix, 0); + Matrix.setIdentityM(mMMatrix, 0); + + float[] defaultAcceleration = {0.f,0.f,0.f}; + setAcceleration(defaultAcceleration); + mPos[0] = 0.f; + mPos[1] = 0.f; + mPos[2] = 0.f; + mVel[0] = 0.f; + mVel[1] = 0.f; + mVel[2] = 0.f; + + } + + /* The following set methods are not synchronized, so should only + * be called within the rendering thread context. Use GLSurfaceView.queueEvent for safe access. + */ + public void setPosition(float x, float y) { + /* Map from screen (0,0)-(1,1) to scene coordinates */ + mPos[0] = (x*2-1)*mRatio; + mPos[1] = (-y)*2+1; + mPos[2] = 0.f; + mVel[0] = 0; + mVel[1] = 0; + mVel[2] = 0; + } + + public void setCamera(Camera camera) { + mCamera = camera; + Camera.Size previewSize = camera.getParameters().getPreviewSize(); + mCameraRatio = (float)previewSize.width/previewSize.height; + } + + public void setAcceleration(float[] accelerationVector) { + mGForce[0] = accelerationVector[0]; + mGForce[1] = accelerationVector[1]; + mGForce[2] = accelerationVector[2]; + } + + public void onDrawFrame(GL10 glUnused) { + synchronized(this) { + if (updateSurface) { + mSurface.updateTexImage(); + + mSurface.getTransformMatrix(mSTMatrix); + long timestamp = mSurface.getTimestamp(); + doPhysics(timestamp); + + updateSurface = false; + } + } + + // Ignore the passed-in GL10 interface, and use the GLES20 + // class's static methods instead. + GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT); + GLES20.glUseProgram(mProgram); + checkGlError("glUseProgram"); + + GLES20.glActiveTexture(GLES20.GL_TEXTURE0); + GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID); + + mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET); + GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false, + TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); + checkGlError("glVertexAttribPointer maPosition"); + GLES20.glEnableVertexAttribArray(maPositionHandle); + checkGlError("glEnableVertexAttribArray maPositionHandle"); + + mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET); + GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false, + TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices); + checkGlError("glVertexAttribPointer maTextureHandle"); + GLES20.glEnableVertexAttribArray(maTextureHandle); + checkGlError("glEnableVertexAttribArray maTextureHandle"); + + Matrix.multiplyMM(mMVPMatrix, 0, mVMatrix, 0, mMMatrix, 0); + Matrix.multiplyMM(mMVPMatrix, 0, mProjMatrix, 0, mMVPMatrix, 0); + + GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0); + GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0); + GLES20.glUniform1f(muCRatioHandle, mCameraRatio); + + GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); + checkGlError("glDrawArrays"); + } + + public void onSurfaceChanged(GL10 glUnused, int width, int height) { + // Ignore the passed-in GL10 interface, and use the GLES20 + // class's static methods instead. + GLES20.glViewport(0, 0, width, height); + mRatio = (float) width / height; + Matrix.frustumM(mProjMatrix, 0, -mRatio, mRatio, -1, 1, 3, 7); + } + + public void onSurfaceCreated(GL10 glUnused, EGLConfig config) { + // Ignore the passed-in GL10 interface, and use the GLES20 + // class's static methods instead. + + /* Set up alpha blending and an Android background color */ + GLES20.glEnable(GLES20.GL_BLEND); + GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA); + GLES20.glClearColor(0.643f, 0.776f, 0.223f, 1.0f); + + /* Set up shaders and handles to their variables */ + mProgram = createProgram(mVertexShader, mFragmentShader); + if (mProgram == 0) { + return; + } + maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition"); + checkGlError("glGetAttribLocation aPosition"); + if (maPositionHandle == -1) { + throw new RuntimeException("Could not get attrib location for aPosition"); + } + maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord"); + checkGlError("glGetAttribLocation aTextureCoord"); + if (maTextureHandle == -1) { + throw new RuntimeException("Could not get attrib location for aTextureCoord"); + } + + muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix"); + checkGlError("glGetUniformLocation uMVPMatrix"); + if (muMVPMatrixHandle == -1) { + throw new RuntimeException("Could not get attrib location for uMVPMatrix"); + } + + muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix"); + checkGlError("glGetUniformLocation uSTMatrix"); + if (muMVPMatrixHandle == -1) { + throw new RuntimeException("Could not get attrib location for uSTMatrix"); + } + + muCRatioHandle = GLES20.glGetUniformLocation(mProgram, "uCRatio"); + checkGlError("glGetUniformLocation uCRatio"); + if (muMVPMatrixHandle == -1) { + throw new RuntimeException("Could not get attrib location for uCRatio"); + } + + /* + * Create our texture. This has to be done each time the + * surface is created. + */ + + int[] textures = new int[1]; + GLES20.glGenTextures(1, textures, 0); + + mTextureID = textures[0]; + GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID); + checkGlError("glBindTexture mTextureID"); + + // Can't do mipmapping with camera source + GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, + GLES20.GL_NEAREST); + GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, + GLES20.GL_LINEAR); + // Clamp to edge is the only option + GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, + GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, + GLES20.GL_CLAMP_TO_EDGE); + checkGlError("glTexParameteri mTextureID"); + + /* + * Create the SurfaceTexture that will feed this textureID, and pass it to the camera + */ + + mSurface = new SurfaceTexture(mTextureID); + mSurface.setOnFrameAvailableListener(this); + try { + mCamera.setPreviewTexture(mSurface); + } catch (IOException t) { + Log.e(TAG, "Cannot set preview texture target!"); + } + + /* Start the camera */ + mCamera.startPreview(); + + Matrix.setLookAtM(mVMatrix, 0, 0, 0, 5f, 0f, 0f, 0f, 0f, 1.0f, 0.0f); + + mLastTime = 0; + + synchronized(this) { + updateSurface = false; + } + } + + synchronized public void onFrameAvailable(SurfaceTexture surface) { + /* For simplicity, SurfaceTexture calls here when it has new + * data available. Call may come in from some random thread, + * so let's be safe and use synchronize. No OpenGL calls can be done here. + */ + updateSurface = true; + } + + private void doPhysics(long timestamp) { + /* + * Move the camera surface around based on some simple spring physics with drag + */ + + if (mLastTime == 0) + mLastTime = timestamp; + + float deltaT = (timestamp - mLastTime)/1000000000.f; // To seconds + + float springStrength = 20.f; + float frictionCoeff = 10.f; + float mass = 10.f; + float gMultiplier = 4.f; + /* Only update physics every 30 ms */ + if (deltaT > 0.030f) { + mLastTime = timestamp; + + float[] totalForce = new float[3]; + totalForce[0] = -mPos[0] * springStrength - mVel[0]*frictionCoeff + gMultiplier*mGForce[0]*mass; + totalForce[1] = -mPos[1] * springStrength - mVel[1]*frictionCoeff + gMultiplier*mGForce[1]*mass; + totalForce[2] = -mPos[2] * springStrength - mVel[2]*frictionCoeff + gMultiplier*mGForce[2]*mass; + + float[] accel = new float[3]; + accel[0] = totalForce[0]/mass; + accel[1] = totalForce[1]/mass; + accel[2] = totalForce[2]/mass; + + /* Not a very accurate integrator */ + mVel[0] = mVel[0] + accel[0]*deltaT; + mVel[1] = mVel[1] + accel[1]*deltaT; + mVel[2] = mVel[2] + accel[2]*deltaT; + + mPos[0] = mPos[0] + mVel[0]*deltaT; + mPos[1] = mPos[1] + mVel[1]*deltaT; + mPos[2] = mPos[2] + mVel[2]*deltaT; + + Matrix.setIdentityM(mMMatrix, 0); + Matrix.translateM(mMMatrix, 0, mPos[0], mPos[1], mPos[2]); + } + + } + + private int loadShader(int shaderType, String source) { + int shader = GLES20.glCreateShader(shaderType); + if (shader != 0) { + GLES20.glShaderSource(shader, source); + GLES20.glCompileShader(shader); + int[] compiled = new int[1]; + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0); + if (compiled[0] == 0) { + Log.e(TAG, "Could not compile shader " + shaderType + ":"); + Log.e(TAG, GLES20.glGetShaderInfoLog(shader)); + GLES20.glDeleteShader(shader); + shader = 0; + } + } + return shader; + } + + private int createProgram(String vertexSource, String fragmentSource) { + int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource); + if (vertexShader == 0) { + return 0; + } + int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource); + if (pixelShader == 0) { + return 0; + } + + int program = GLES20.glCreateProgram(); + if (program != 0) { + GLES20.glAttachShader(program, vertexShader); + checkGlError("glAttachShader"); + GLES20.glAttachShader(program, pixelShader); + checkGlError("glAttachShader"); + GLES20.glLinkProgram(program); + int[] linkStatus = new int[1]; + GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0); + if (linkStatus[0] != GLES20.GL_TRUE) { + Log.e(TAG, "Could not link program: "); + Log.e(TAG, GLES20.glGetProgramInfoLog(program)); + GLES20.glDeleteProgram(program); + program = 0; + } + } + return program; + } + + private void checkGlError(String op) { + int error; + while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { + Log.e(TAG, op + ": glError " + error); + throw new RuntimeException(op + ": glError " + error); + } + } + + private static final int FLOAT_SIZE_BYTES = 4; + private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES; + private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0; + private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3; + private final float[] mTriangleVerticesData = { + // X, Y, Z, U, V + -1.0f, -1.0f, 0, 0.f, 0.f, + 1.0f, -1.0f, 0, 1.f, 0.f, + -1.0f, 1.0f, 0, 0.f, 1.f, + 1.0f, 1.0f, 0, 1.f, 1.f, + }; + + private FloatBuffer mTriangleVertices; + + private final String mVertexShader = + "uniform mat4 uMVPMatrix;\n" + + "uniform mat4 uSTMatrix;\n" + + "uniform float uCRatio;\n" + + "attribute vec4 aPosition;\n" + + "attribute vec4 aTextureCoord;\n" + + "varying vec2 vTextureCoord;\n" + + "varying vec2 vTextureNormCoord;\n" + + "void main() {\n" + + " vec4 scaledPos = aPosition;\n" + + " scaledPos.x = scaledPos.x * uCRatio;\n" + + " gl_Position = uMVPMatrix * scaledPos;\n" + + " vTextureCoord = (uSTMatrix * aTextureCoord).xy;\n" + + " vTextureNormCoord = aTextureCoord.xy;\n" + + "}\n"; + + private final String mFragmentShader = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "varying vec2 vTextureCoord;\n" + + "varying vec2 vTextureNormCoord;\n" + + "uniform samplerExternalOES sTexture;\n" + + "void main() {\n" + + " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + + " gl_FragColor.a = 1.0-min(length(vTextureNormCoord-0.5)*2.0,1.0);\n" + + "}\n"; + + private float[] mMVPMatrix = new float[16]; + private float[] mProjMatrix = new float[16]; + private float[] mMMatrix = new float[16]; + private float[] mVMatrix = new float[16]; + private float[] mSTMatrix = new float[16]; + + private int mProgram; + private int mTextureID; + private int muMVPMatrixHandle; + private int muSTMatrixHandle; + private int muCRatioHandle; + private int maPositionHandle; + private int maTextureHandle; + + private float mRatio = 1.0f; + private float mCameraRatio = 1.0f; + private float[] mVel = new float[3]; + private float[] mPos = new float[3]; + private float[] mGForce = new float[3]; + + private long mLastTime; + + private SurfaceTexture mSurface; + private Camera mCamera; + private boolean updateSurface = false; + + private Context mContext; + private static String TAG = "CamRenderer"; + + // Magic key + private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65; +}