From ff2ed70fa30f04b90dd1a2c06ec2319e157152d7 Mon Sep 17 00:00:00 2001 From: Mathias Agopian Date: Sun, 1 Sep 2013 21:36:12 -0700 Subject: [PATCH] color blindness enhancement This is an attempt at improving the experience of users with color vision impairement. At this time this feature can only be enabled for debugging: adb shell service call SurfaceFlinger 1014 i32 PARAM with PARAM: 0 : disabled 1 : protanomaly/protanopia simulation 2 : deuteranomaly/deuteranopia simulation 3 : tritanopia/tritanomaly simulation 11, 12, 13: same as above w/ attempted correction/enhancement The enhancement algorithm tries to spread the "error" such that tones that would otherwise appear similar can be distinguished. Bug: 9465644 Change-Id: I860f7eed0cb81f54ef9cf24ad78155b6395ade48 --- services/surfaceflinger/Android.mk | 1 + .../surfaceflinger/Effects/Daltonizer.cpp | 183 ++++++++++++++++++ services/surfaceflinger/Effects/Daltonizer.h | 59 ++++++ services/surfaceflinger/Layer.cpp | 24 +-- .../RenderEngine/Description.cpp | 10 +- .../surfaceflinger/RenderEngine/Description.h | 4 + .../RenderEngine/GLES11RenderEngine.cpp | 8 + .../RenderEngine/GLES11RenderEngine.h | 3 + .../RenderEngine/GLES20RenderEngine.cpp | 82 +++++++- .../RenderEngine/GLES20RenderEngine.h | 14 ++ services/surfaceflinger/RenderEngine/Mesh.h | 30 ++- .../surfaceflinger/RenderEngine/Program.cpp | 4 + .../surfaceflinger/RenderEngine/Program.h | 3 + .../RenderEngine/ProgramCache.cpp | 22 ++- .../RenderEngine/ProgramCache.h | 7 + .../RenderEngine/RenderEngine.cpp | 2 +- .../RenderEngine/RenderEngine.h | 7 + services/surfaceflinger/SurfaceFlinger.cpp | 37 +++- services/surfaceflinger/SurfaceFlinger.h | 4 +- services/surfaceflinger/Transform.cpp | 15 +- services/surfaceflinger/Transform.h | 24 +-- 21 files changed, 472 insertions(+), 71 deletions(-) create mode 100644 services/surfaceflinger/Effects/Daltonizer.cpp create mode 100644 services/surfaceflinger/Effects/Daltonizer.h diff --git a/services/surfaceflinger/Android.mk b/services/surfaceflinger/Android.mk index 3888d7e54..7cc4ce195 100644 --- a/services/surfaceflinger/Android.mk +++ b/services/surfaceflinger/Android.mk @@ -17,6 +17,7 @@ LOCAL_SRC_FILES:= \ DisplayHardware/HWComposer.cpp \ DisplayHardware/PowerHAL.cpp \ DisplayHardware/VirtualDisplaySurface.cpp \ + Effects/Daltonizer.cpp \ EventLog/EventLogTags.logtags \ EventLog/EventLog.cpp \ RenderEngine/Description.cpp \ diff --git a/services/surfaceflinger/Effects/Daltonizer.cpp b/services/surfaceflinger/Effects/Daltonizer.cpp new file mode 100644 index 000000000..f384ba453 --- /dev/null +++ b/services/surfaceflinger/Effects/Daltonizer.cpp @@ -0,0 +1,183 @@ +/* + * Copyright 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Daltonizer.h" +#include + +namespace android { + +Daltonizer::Daltonizer() : + mType(deuteranomaly), mMode(simulation), mDirty(true) { +} + +Daltonizer::~Daltonizer() { +} + +void Daltonizer::setType(Daltonizer::ColorBlindnessTypes type) { + if (type != mType) { + mDirty = true; + mType = type; + } +} + +void Daltonizer::setMode(Daltonizer::Mode mode) { + if (mode != mMode) { + mDirty = true; + mMode = mode; + } +} + +const mat4& Daltonizer::operator()() { + if (mDirty) { + mDirty = false; + update(); + } + return mColorTransform; +} + +void Daltonizer::update() { + // converts a linear RGB color to the XYZ space + const mat4 rgb2xyz( 0.4124, 0.2126, 0.0193, 0, + 0.3576, 0.7152, 0.1192, 0, + 0.1805, 0.0722, 0.9505, 0, + 0 , 0 , 0 , 1); + + // converts a XYZ color to the LMS space. + const mat4 xyz2lms( 0.7328,-0.7036, 0.0030, 0, + 0.4296, 1.6975, 0.0136, 0, + -0.1624, 0.0061, 0.9834, 0, + 0 , 0 , 0 , 1); + + // Direct conversion from linear RGB to LMS + const mat4 rgb2lms(xyz2lms*rgb2xyz); + + // And back from LMS to linear RGB + const mat4 lms2rgb(inverse(rgb2lms)); + + // To simulate color blindness we need to "remove" the data lost by the absence of + // a cone. This cannot be done by just zeroing out the corresponding LMS component + // because it would create a color outside of the RGB gammut. + // Instead we project the color along the axis of the missing component onto a plane + // within the RGB gammut: + // - since the projection happens along the axis of the missing component, a + // color blind viewer perceives the projected color the same. + // - We use the plane defined by 3 points in LMS space: black, white and + // blue and red for protanopia/deuteranopia and tritanopia respectively. + + // LMS space red + const vec3& lms_r(rgb2lms[0].rgb); + // LMS space blue + const vec3& lms_b(rgb2lms[2].rgb); + // LMS space white + const vec3 lms_w((rgb2lms * vec4(1)).rgb); + + // To find the planes we solve the a*L + b*M + c*S = 0 equation for the LMS values + // of the three known points. This equation is trivially solved, and has for + // solution the following cross-products: + const vec3 p0 = cross(lms_w, lms_b); // protanopia/deuteranopia + const vec3 p1 = cross(lms_w, lms_r); // tritanopia + + // The following 3 matrices perform the projection of a LMS color onto the given plane + // along the selected axis + + // projection for protanopia (L = 0) + const mat4 lms2lmsp( 0.0000, 0.0000, 0.0000, 0, + -p0.y / p0.x, 1.0000, 0.0000, 0, + -p0.z / p0.x, 0.0000, 1.0000, 0, + 0 , 0 , 0 , 1); + + // projection for deuteranopia (M = 0) + const mat4 lms2lmsd( 1.0000, -p0.x / p0.y, 0.0000, 0, + 0.0000, 0.0000, 0.0000, 0, + 0.0000, -p0.z / p0.y, 1.0000, 0, + 0 , 0 , 0 , 1); + + // projection for tritanopia (S = 0) + const mat4 lms2lmst( 1.0000, 0.0000, -p1.x / p1.z, 0, + 0.0000, 1.0000, -p1.y / p1.z, 0, + 0.0000, 0.0000, 0.0000, 0, + 0 , 0 , 0 , 1); + + // We will calculate the error between the color and the color viewed by + // a color blind user and "spread" this error onto the healthy cones. + // The matrices below perform this last step and have been chosen arbitrarily. + + // The amount of correction can be adjusted here. + + // error spread for protanopia + const mat4 errp( 1.0, 0.7, 0.7, 0, + 0.0, 1.0, 0.0, 0, + 0.0, 0.0, 1.0, 0, + 0, 0, 0, 1); + + // error spread for deuteranopia + const mat4 errd( 1.0, 0.0, 0.0, 0, + 0.7, 1.0, 0.7, 0, + 0.0, 0.0, 1.0, 0, + 0, 0, 0, 1); + + // error spread for tritanopia + const mat4 errt( 1.0, 0.0, 0.0, 0, + 0.0, 1.0, 0.0, 0, + 0.7, 0.7, 1.0, 0, + 0, 0, 0, 1); + + const mat4 identity; + + // And the magic happens here... + // We construct the matrix that will perform the whole correction. + + // simulation: type of color blindness to simulate: + // set to either lms2lmsp, lms2lmsd, lms2lmst + mat4 simulation; + + // correction: type of color blindness correction (should match the simulation above): + // set to identity, errp, errd, errt ([0] for simulation only) + mat4 correction(0); + + // control: simulation post-correction (used for debugging): + // set to identity or lms2lmsp, lms2lmsd, lms2lmst + mat4 control; + switch (mType) { + case protanopia: + case protanomaly: + simulation = lms2lmsp; + if (mMode == Daltonizer::correction) + correction = errp; + break; + case deuteranopia: + case deuteranomaly: + simulation = lms2lmsd; + if (mMode == Daltonizer::correction) + correction = errd; + break; + case tritanopia: + case tritanomaly: + simulation = lms2lmst; + if (mMode == Daltonizer::correction) + correction = errt; + break; + } + + if (true) { + control = simulation; + } + + mColorTransform = lms2rgb * control * + (simulation * rgb2lms + correction * (rgb2lms - simulation * rgb2lms)); +} + +} /* namespace android */ diff --git a/services/surfaceflinger/Effects/Daltonizer.h b/services/surfaceflinger/Effects/Daltonizer.h new file mode 100644 index 000000000..e81643747 --- /dev/null +++ b/services/surfaceflinger/Effects/Daltonizer.h @@ -0,0 +1,59 @@ +/* + * Copyright 2013 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef SF_EFFECTS_DALTONIZER_H_ +#define SF_EFFECTS_DALTONIZER_H_ + +#include + +namespace android { + +class Daltonizer { +public: + enum ColorBlindnessTypes { + protanopia, // L (red) cone missing + deuteranopia, // M (green) cone missing + tritanopia, // S (blue) cone missing + protanomaly, // L (red) cone deficient + deuteranomaly, // M (green) cone deficient (most common) + tritanomaly // S (blue) cone deficient + }; + + enum Mode { + simulation, + correction + }; + + Daltonizer(); + ~Daltonizer(); + + void setType(ColorBlindnessTypes type); + void setMode(Mode mode); + + // returns the color transform to apply in the shader + const mat4& operator()(); + +private: + void update(); + + ColorBlindnessTypes mType; + Mode mMode; + bool mDirty; + mat4 mColorTransform; +}; + +} /* namespace android */ +#endif /* SF_EFFECTS_DALTONIZER_H_ */ diff --git a/services/surfaceflinger/Layer.cpp b/services/surfaceflinger/Layer.cpp index 56bddd61e..b610c2066 100644 --- a/services/surfaceflinger/Layer.cpp +++ b/services/surfaceflinger/Layer.cpp @@ -547,15 +547,11 @@ void Layer::drawWithOpenGL( // TODO: we probably want to generate the texture coords with the mesh // here we assume that we only have 4 vertices - Mesh::VertexArray texCoords(mMesh.getTexCoordArray()); - texCoords[0].s = left; - texCoords[0].t = 1.0f - top; - texCoords[1].s = left; - texCoords[1].t = 1.0f - bottom; - texCoords[2].s = right; - texCoords[2].t = 1.0f - bottom; - texCoords[3].s = right; - texCoords[3].t = 1.0f - top; + Mesh::VertexArray texCoords(mMesh.getTexCoordArray()); + texCoords[0] = vec2(left, 1.0f - top); + texCoords[1] = vec2(left, 1.0f - bottom); + texCoords[2] = vec2(right, 1.0f - bottom); + texCoords[3] = vec2(right, 1.0f - top); RenderEngine& engine(mFlinger->getRenderEngine()); engine.setupLayerBlending(mPremultipliedAlpha, isOpaque(), s.alpha); @@ -608,11 +604,11 @@ void Layer::computeGeometry(const sp& hw, Mesh& mesh) const // subtract the transparent region and snap to the bounds win = reduce(win, s.activeTransparentRegion); - Mesh::VertexArray position(mesh.getPositionArray()); - tr.transform(position[0], win.left, win.top); - tr.transform(position[1], win.left, win.bottom); - tr.transform(position[2], win.right, win.bottom); - tr.transform(position[3], win.right, win.top); + Mesh::VertexArray position(mesh.getPositionArray()); + position[0] = tr.transform(win.left, win.top); + position[1] = tr.transform(win.left, win.bottom); + position[2] = tr.transform(win.right, win.bottom); + position[3] = tr.transform(win.right, win.top); for (size_t i=0 ; i<4 ; i++) { position[i].y = hw_h - position[i].y; } diff --git a/services/surfaceflinger/RenderEngine/Description.cpp b/services/surfaceflinger/RenderEngine/Description.cpp index b0325c67d..1adcd1f5a 100644 --- a/services/surfaceflinger/RenderEngine/Description.cpp +++ b/services/surfaceflinger/RenderEngine/Description.cpp @@ -29,9 +29,10 @@ namespace android { Description::Description() : mUniformsDirty(true) { mPlaneAlpha = 1.0f; - mPremultipliedAlpha = true; + mPremultipliedAlpha = false; mOpaque = true; mTextureEnabled = false; + mColorMatrixEnabled = false; memset(mColor, 0, sizeof(mColor)); } @@ -81,4 +82,11 @@ void Description::setProjectionMatrix(const mat4& mtx) { mUniformsDirty = true; } +void Description::setColorMatrix(const mat4& mtx) { + const mat4 identity; + mColorMatrix = mtx; + mColorMatrixEnabled = (mtx != identity); +} + + } /* namespace android */ diff --git a/services/surfaceflinger/RenderEngine/Description.h b/services/surfaceflinger/RenderEngine/Description.h index 0230762bb..43b835f02 100644 --- a/services/surfaceflinger/RenderEngine/Description.h +++ b/services/surfaceflinger/RenderEngine/Description.h @@ -51,6 +51,9 @@ class Description { // projection matrix mat4 mProjectionMatrix; + bool mColorMatrixEnabled; + mat4 mColorMatrix; + public: Description(); ~Description(); @@ -62,6 +65,7 @@ public: void disableTexture(); void setColor(GLclampf red, GLclampf green, GLclampf blue, GLclampf alpha); void setProjectionMatrix(const mat4& mtx); + void setColorMatrix(const mat4& mtx); private: bool mUniformsDirty; diff --git a/services/surfaceflinger/RenderEngine/GLES11RenderEngine.cpp b/services/surfaceflinger/RenderEngine/GLES11RenderEngine.cpp index d0ae25350..1cd8cb34c 100644 --- a/services/surfaceflinger/RenderEngine/GLES11RenderEngine.cpp +++ b/services/surfaceflinger/RenderEngine/GLES11RenderEngine.cpp @@ -237,6 +237,14 @@ void GLES11RenderEngine::drawMesh(const Mesh& mesh) { } } +void GLES11RenderEngine::beginGroup(const mat4& colorTransform) { + // doesn't do anything in GLES 1.1 +} + +void GLES11RenderEngine::endGroup() { + // doesn't do anything in GLES 1.1 +} + void GLES11RenderEngine::dump(String8& result) { RenderEngine::dump(result); } diff --git a/services/surfaceflinger/RenderEngine/GLES11RenderEngine.h b/services/surfaceflinger/RenderEngine/GLES11RenderEngine.h index 1de0443af..cd53aab38 100644 --- a/services/surfaceflinger/RenderEngine/GLES11RenderEngine.h +++ b/services/surfaceflinger/RenderEngine/GLES11RenderEngine.h @@ -60,6 +60,9 @@ protected: virtual void drawMesh(const Mesh& mesh); + virtual void beginGroup(const mat4& colorTransform); + virtual void endGroup(); + virtual size_t getMaxTextureSize() const; virtual size_t getMaxViewportDims() const; }; diff --git a/services/surfaceflinger/RenderEngine/GLES20RenderEngine.cpp b/services/surfaceflinger/RenderEngine/GLES20RenderEngine.cpp index 9754758ca..7112ca8d2 100644 --- a/services/surfaceflinger/RenderEngine/GLES20RenderEngine.cpp +++ b/services/surfaceflinger/RenderEngine/GLES20RenderEngine.cpp @@ -35,7 +35,8 @@ namespace android { // --------------------------------------------------------------------------- -GLES20RenderEngine::GLES20RenderEngine() { +GLES20RenderEngine::GLES20RenderEngine() : + mVpWidth(0), mVpHeight(0) { glGetIntegerv(GL_MAX_TEXTURE_SIZE, &mMaxTextureSize); glGetIntegerv(GL_MAX_VIEWPORT_DIMS, mMaxViewportDims); @@ -58,6 +59,8 @@ GLES20RenderEngine::GLES20RenderEngine() { glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, 1, 1, 0, GL_RGB, GL_UNSIGNED_SHORT_5_6_5, protTexData); + + //mColorBlindnessCorrection = M; } GLES20RenderEngine::~GLES20RenderEngine() { @@ -82,6 +85,8 @@ void GLES20RenderEngine::setViewportAndProjection( glViewport(0, 0, vpw, vph); mState.setProjectionMatrix(m); + mVpWidth = vpw; + mVpHeight = vph; } void GLES20RenderEngine::setupLayerBlending( @@ -156,8 +161,7 @@ void GLES20RenderEngine::bindImageAsFramebuffer(EGLImageKHR image, // create a Framebuffer Object to render into glGenFramebuffers(1, &name); glBindFramebuffer(GL_FRAMEBUFFER, name); - glFramebufferTexture2D(GL_FRAMEBUFFER, - GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, tname, 0); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, tname, 0); *status = glCheckFramebufferStatus(GL_FRAMEBUFFER); *texName = tname; @@ -205,6 +209,78 @@ void GLES20RenderEngine::drawMesh(const Mesh& mesh) { } } +void GLES20RenderEngine::beginGroup(const mat4& colorTransform) { + + GLuint tname, name; + // create the texture + glGenTextures(1, &tname); + glBindTexture(GL_TEXTURE_2D, tname); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, mVpWidth, mVpHeight, 0, GL_RGBA, GL_UNSIGNED_BYTE, 0); + + // create a Framebuffer Object to render into + glGenFramebuffers(1, &name); + glBindFramebuffer(GL_FRAMEBUFFER, name); + glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, tname, 0); + + Group group; + group.texture = tname; + group.fbo = name; + group.width = mVpWidth; + group.height = mVpHeight; + group.colorTransform = colorTransform; + + mGroupStack.push(group); +} + +void GLES20RenderEngine::endGroup() { + + const Group group(mGroupStack.top()); + mGroupStack.pop(); + + // activate the previous render target + GLuint fbo = 0; + if (!mGroupStack.isEmpty()) { + fbo = mGroupStack.top().fbo; + } + glBindFramebuffer(GL_FRAMEBUFFER, fbo); + + // set our state + Texture texture(Texture::TEXTURE_2D, group.texture); + texture.setDimensions(group.width, group.height); + glBindTexture(GL_TEXTURE_2D, group.texture); + + mState.setPlaneAlpha(1.0f); + mState.setPremultipliedAlpha(true); + mState.setOpaque(false); + mState.setTexture(texture); + mState.setColorMatrix(group.colorTransform); + glDisable(GL_BLEND); + + Mesh mesh(Mesh::TRIANGLE_FAN, 4, 2, 2); + Mesh::VertexArray position(mesh.getPositionArray()); + Mesh::VertexArray texCoord(mesh.getTexCoordArray()); + position[0] = vec2(0, 0); + position[1] = vec2(group.width, 0); + position[2] = vec2(group.width, group.height); + position[3] = vec2(0, group.height); + texCoord[0] = vec2(0, 0); + texCoord[1] = vec2(1, 0); + texCoord[2] = vec2(1, 1); + texCoord[3] = vec2(0, 1); + drawMesh(mesh); + + // reset color matrix + mState.setColorMatrix(mat4()); + + // free our fbo and texture + glDeleteFramebuffers(1, &group.fbo); + glDeleteTextures(1, &group.texture); +} + void GLES20RenderEngine::dump(String8& result) { RenderEngine::dump(result); } diff --git a/services/surfaceflinger/RenderEngine/GLES20RenderEngine.h b/services/surfaceflinger/RenderEngine/GLES20RenderEngine.h index eb8f31afb..8b67fccdc 100644 --- a/services/surfaceflinger/RenderEngine/GLES20RenderEngine.h +++ b/services/surfaceflinger/RenderEngine/GLES20RenderEngine.h @@ -39,8 +39,19 @@ class GLES20RenderEngine : public RenderEngine { GLuint mProtectedTexName; GLint mMaxViewportDims[2]; GLint mMaxTextureSize; + GLuint mVpWidth; + GLuint mVpHeight; + + struct Group { + GLuint texture; + GLuint fbo; + GLuint width; + GLuint height; + mat4 colorTransform; + }; Description mState; + Vector mGroupStack; virtual void bindImageAsFramebuffer(EGLImageKHR image, uint32_t* texName, uint32_t* fbName, uint32_t* status); @@ -64,6 +75,9 @@ protected: virtual void drawMesh(const Mesh& mesh); + virtual void beginGroup(const mat4& colorTransform); + virtual void endGroup(); + virtual size_t getMaxTextureSize() const; virtual size_t getMaxViewportDims() const; }; diff --git a/services/surfaceflinger/RenderEngine/Mesh.h b/services/surfaceflinger/RenderEngine/Mesh.h index 160d765ba..b6d42b097 100644 --- a/services/surfaceflinger/RenderEngine/Mesh.h +++ b/services/surfaceflinger/RenderEngine/Mesh.h @@ -33,32 +33,28 @@ public: ~Mesh(); /* - * VertexArray handles the stride automatically. It also provides - * a convenient way to set position and texture coordinates by using - * the usual x,y,z,w or s,t,r,q names. + * VertexArray handles the stride automatically. */ + template class VertexArray { friend class Mesh; float* mData; size_t mStride; VertexArray(float* data, size_t stride) : mData(data), mStride(stride) { } public: - struct vertexData { - operator float*() { return reinterpret_cast(this); } - union { - struct { float x, y, z, w; }; - struct { float s, t, r, q; }; - }; - }; - vertexData& operator[](size_t index) { - return *reinterpret_cast(&mData[index*mStride]); } - - vertexData const& operator[](size_t index) const { - return *reinterpret_cast(&mData[index*mStride]); } + TYPE& operator[](size_t index) { + return *reinterpret_cast(&mData[index*mStride]); + } + TYPE const& operator[](size_t index) const { + return *reinterpret_cast(&mData[index*mStride]); + } }; - VertexArray getPositionArray() { return VertexArray(getPositions(), mStride); } - VertexArray getTexCoordArray() { return VertexArray(getTexCoords(), mStride); } + template + VertexArray getPositionArray() { return VertexArray(getPositions(), mStride); } + + template + VertexArray getTexCoordArray() { return VertexArray(getTexCoords(), mStride); } Primitive getPrimitive() const; diff --git a/services/surfaceflinger/RenderEngine/Program.cpp b/services/surfaceflinger/RenderEngine/Program.cpp index ece09050a..4a7fb58d2 100644 --- a/services/surfaceflinger/RenderEngine/Program.cpp +++ b/services/surfaceflinger/RenderEngine/Program.cpp @@ -58,6 +58,7 @@ Program::Program(const ProgramCache::Key& needs, const char* vertex, const char* mFragmentShader = fragmentId; mInitialized = true; + mColorMatrixLoc = glGetUniformLocation(programId, "colorMatrix"); mProjectionMatrixLoc = glGetUniformLocation(programId, "projection"); mTextureMatrixLoc = glGetUniformLocation(programId, "texture"); mSamplerLoc = glGetUniformLocation(programId, "sampler"); @@ -137,6 +138,9 @@ void Program::setUniforms(const Description& desc) { if (mColorLoc >= 0) { glUniform4fv(mColorLoc, 1, desc.mColor); } + if (mColorMatrixLoc >= 0) { + glUniformMatrix4fv(mColorMatrixLoc, 1, GL_FALSE, desc.mColorMatrix.asArray()); + } // these uniforms are always present glUniformMatrix4fv(mProjectionMatrixLoc, 1, GL_FALSE, desc.mProjectionMatrix.asArray()); } diff --git a/services/surfaceflinger/RenderEngine/Program.h b/services/surfaceflinger/RenderEngine/Program.h index 91bb3dbbc..36bd120e3 100644 --- a/services/surfaceflinger/RenderEngine/Program.h +++ b/services/surfaceflinger/RenderEngine/Program.h @@ -70,6 +70,9 @@ private: /* location of the projection matrix uniform */ GLint mProjectionMatrixLoc; + /* location of the color matrix uniform */ + GLint mColorMatrixLoc; + /* location of the texture matrix uniform */ GLint mTextureMatrixLoc; diff --git a/services/surfaceflinger/RenderEngine/ProgramCache.cpp b/services/surfaceflinger/RenderEngine/ProgramCache.cpp index 62d2eab99..09b0ddc4a 100644 --- a/services/surfaceflinger/RenderEngine/ProgramCache.cpp +++ b/services/surfaceflinger/RenderEngine/ProgramCache.cpp @@ -96,7 +96,9 @@ ProgramCache::Key ProgramCache::computeKey(const Description& description) { .set(Key::BLEND_MASK, description.mPremultipliedAlpha ? Key::BLEND_PREMULT : Key::BLEND_NORMAL) .set(Key::OPACITY_MASK, - description.mOpaque ? Key::OPACITY_OPAQUE : Key::OPACITY_TRANSLUCENT); + description.mOpaque ? Key::OPACITY_OPAQUE : Key::OPACITY_TRANSLUCENT) + .set(Key::COLOR_MATRIX_MASK, + description.mColorMatrixEnabled ? Key::COLOR_MATRIX_ON : Key::COLOR_MATRIX_OFF); return needs; } @@ -139,6 +141,9 @@ String8 ProgramCache::generateFragmentShader(const Key& needs) { if (needs.hasPlaneAlpha()) { fs << "uniform float alphaPlane;"; } + if (needs.hasColorMatrix()) { + fs << "uniform mat4 colorMatrix;"; + } fs << "void main(void) {" << indent; if (needs.isTexturing()) { fs << "gl_FragColor = texture2D(sampler, outTexCoords);"; @@ -157,6 +162,21 @@ String8 ProgramCache::generateFragmentShader(const Key& needs) { fs << "gl_FragColor.a *= alphaPlane;"; } } + + if (needs.hasColorMatrix()) { + if (!needs.isOpaque() && needs.isPremultiplied()) { + // un-premultiply if needed before linearization + fs << "gl_FragColor.rgb = gl_FragColor.rgb/gl_FragColor.a;"; + } + fs << "gl_FragColor.rgb = pow(gl_FragColor.rgb, vec3(2.2));"; + fs << "gl_FragColor = colorMatrix*gl_FragColor;"; + fs << "gl_FragColor.rgb = pow(gl_FragColor.rgb, vec3(1.0 / 2.2));"; + if (!needs.isOpaque() && needs.isPremultiplied()) { + // and re-premultiply if needed after gamma correction + fs << "gl_FragColor.rgb = gl_FragColor.rgb*gl_FragColor.a;"; + } + } + fs << dedent << "}"; return fs.getString(); } diff --git a/services/surfaceflinger/RenderEngine/ProgramCache.h b/services/surfaceflinger/RenderEngine/ProgramCache.h index fcbeffd30..e8b9dcd78 100644 --- a/services/surfaceflinger/RenderEngine/ProgramCache.h +++ b/services/surfaceflinger/RenderEngine/ProgramCache.h @@ -65,6 +65,10 @@ public: TEXTURE_EXT = 0x00000008, TEXTURE_2D = 0x00000010, TEXTURE_MASK = 0x00000018, + + COLOR_MATRIX_OFF = 0x00000000, + COLOR_MATRIX_ON = 0x00000020, + COLOR_MATRIX_MASK = 0x00000020, }; inline Key() : mKey(0) { } @@ -90,6 +94,9 @@ public: inline bool hasPlaneAlpha() const { return (mKey & PLANE_ALPHA_MASK) == PLANE_ALPHA_LT_ONE; } + inline bool hasColorMatrix() const { + return (mKey & COLOR_MATRIX_MASK) == COLOR_MATRIX_ON; + } // this is the definition of a friend function -- not a method of class Needs friend inline int strictly_order_type(const Key& lhs, const Key& rhs) { diff --git a/services/surfaceflinger/RenderEngine/RenderEngine.cpp b/services/surfaceflinger/RenderEngine/RenderEngine.cpp index 2abda82dc..46f628dd8 100644 --- a/services/surfaceflinger/RenderEngine/RenderEngine.cpp +++ b/services/surfaceflinger/RenderEngine/RenderEngine.cpp @@ -149,7 +149,7 @@ void RenderEngine::fillRegionWithColor(const Region& region, uint32_t height, size_t c; Rect const* r = region.getArray(&c); Mesh mesh(Mesh::TRIANGLES, c*6, 2); - Mesh::VertexArray position(mesh.getPositionArray()); + Mesh::VertexArray position(mesh.getPositionArray()); for (size_t i=0 ; ileft; position[i*6 + 0].y = height - r->top; diff --git a/services/surfaceflinger/RenderEngine/RenderEngine.h b/services/surfaceflinger/RenderEngine/RenderEngine.h index bc88b69c9..5f331d4a4 100644 --- a/services/surfaceflinger/RenderEngine/RenderEngine.h +++ b/services/surfaceflinger/RenderEngine/RenderEngine.h @@ -23,6 +23,7 @@ #include #include +#include // --------------------------------------------------------------------------- namespace android { @@ -95,6 +96,12 @@ public: // drawing virtual void drawMesh(const Mesh& mesh) = 0; + // grouping + // creates a color-transform group, everything drawn in the group will be + // transformed by the given color transform when endGroup() is called. + virtual void beginGroup(const mat4& colorTransform) = 0; + virtual void endGroup() = 0; + // queries virtual size_t getMaxTextureSize() const = 0; virtual size_t getMaxViewportDims() const = 0; diff --git a/services/surfaceflinger/SurfaceFlinger.cpp b/services/surfaceflinger/SurfaceFlinger.cpp index 0323cb73a..fa1ea0989 100644 --- a/services/surfaceflinger/SurfaceFlinger.cpp +++ b/services/surfaceflinger/SurfaceFlinger.cpp @@ -68,7 +68,10 @@ #include "DisplayHardware/HWComposer.h" #include "DisplayHardware/VirtualDisplaySurface.h" +#include "Effects/Daltonizer.h" + #include "RenderEngine/RenderEngine.h" +#include #define DISPLAY_COUNT 1 @@ -110,7 +113,8 @@ SurfaceFlinger::SurfaceFlinger() mLastSwapBufferTime(0), mDebugInTransaction(0), mLastTransactionTime(0), - mBootFinished(false) + mBootFinished(false), + mDaltonize(false) { ALOGI("SurfaceFlinger is starting"); @@ -865,7 +869,7 @@ void SurfaceFlinger::setUpHWComposer() { for (size_t i=0 ; cur!=end && i& layer(currentLayers[i]); layer->setGeometry(hw, *cur); - if (mDebugDisableHWC || mDebugRegion) { + if (mDebugDisableHWC || mDebugRegion || mDaltonize) { cur->setSkip(true); } } @@ -1479,7 +1483,14 @@ void SurfaceFlinger::doDisplayComposition(const sp& hw, } } - doComposeSurfaces(hw, dirtyRegion); + if (CC_LIKELY(!mDaltonize)) { + doComposeSurfaces(hw, dirtyRegion); + } else { + RenderEngine& engine(getRenderEngine()); + engine.beginGroup(mDaltonizer()); + doComposeSurfaces(hw, dirtyRegion); + engine.endGroup(); + } // update the swap region and clear the dirty region hw->swapRegion.orSelf(dirtyRegion); @@ -2360,7 +2371,7 @@ void SurfaceFlinger::dumpAllLocked(const Vector& args, size_t& index, colorizer.reset(result); result.appendFormat(" h/w composer %s and %s\n", hwc.initCheck()==NO_ERROR ? "present" : "not present", - (mDebugDisableHWC || mDebugRegion) ? "disabled" : "enabled"); + (mDebugDisableHWC || mDebugRegion || mDaltonize) ? "disabled" : "enabled"); hwc.dump(result); /* @@ -2505,6 +2516,24 @@ status_t SurfaceFlinger::onTransact( Mutex::Autolock _l(mStateLock); sp hw(getDefaultDisplayDevice()); reply->writeInt32(hw->getPageFlipCount()); + return NO_ERROR; + } + case 1014: { + // daltonize + n = data.readInt32(); + switch (n % 10) { + case 1: mDaltonizer.setType(Daltonizer::protanomaly); break; + case 2: mDaltonizer.setType(Daltonizer::deuteranomaly); break; + case 3: mDaltonizer.setType(Daltonizer::tritanomaly); break; + } + if (n >= 10) { + mDaltonizer.setMode(Daltonizer::correction); + } else { + mDaltonizer.setMode(Daltonizer::simulation); + } + mDaltonize = n > 0; + invalidateHwcGeometry(); + repaintEverything(); } return NO_ERROR; } diff --git a/services/surfaceflinger/SurfaceFlinger.h b/services/surfaceflinger/SurfaceFlinger.h index f746e6b91..347e3e360 100644 --- a/services/surfaceflinger/SurfaceFlinger.h +++ b/services/surfaceflinger/SurfaceFlinger.h @@ -52,6 +52,7 @@ #include "MessageQueue.h" #include "DisplayHardware/HWComposer.h" +#include "Effects/Daltonizer.h" namespace android { @@ -458,7 +459,8 @@ private: * Feature prototyping */ - sp mExtDisplayToken; + Daltonizer mDaltonizer; + bool mDaltonize; }; // --------------------------------------------------------------------------- diff --git a/services/surfaceflinger/Transform.cpp b/services/surfaceflinger/Transform.cpp index 315720e1c..3456abfa6 100644 --- a/services/surfaceflinger/Transform.cpp +++ b/services/surfaceflinger/Transform.cpp @@ -83,8 +83,8 @@ Transform Transform::operator * (const Transform& rhs) const return r; } -float const* Transform::operator [] (int i) const { - return mMatrix[i].v; +const vec3& Transform::operator [] (size_t i) const { + return mMatrix[i]; } bool Transform::transformed() const { @@ -173,7 +173,7 @@ status_t Transform::set(uint32_t flags, float w, float h) return NO_ERROR; } -Transform::vec2 Transform::transform(const vec2& v) const { +vec2 Transform::transform(const vec2& v) const { vec2 r; const mat33& M(mMatrix); r[0] = M[0][0]*v[0] + M[1][0]*v[1] + M[2][0]; @@ -181,7 +181,7 @@ Transform::vec2 Transform::transform(const vec2& v) const { return r; } -Transform::vec3 Transform::transform(const vec3& v) const { +vec3 Transform::transform(const vec3& v) const { vec3 r; const mat33& M(mMatrix); r[0] = M[0][0]*v[0] + M[1][0]*v[1] + M[2][0]*v[2]; @@ -190,12 +190,9 @@ Transform::vec3 Transform::transform(const vec3& v) const { return r; } -void Transform::transform(float* point, int x, int y) const +vec2 Transform::transform(int x, int y) const { - vec2 v(x, y); - v = transform(v); - point[0] = v[0]; - point[1] = v[1]; + return transform(vec2(x,y)); } Rect Transform::makeBounds(int w, int h) const diff --git a/services/surfaceflinger/Transform.h b/services/surfaceflinger/Transform.h index c4efade76..c2f0010a3 100644 --- a/services/surfaceflinger/Transform.h +++ b/services/surfaceflinger/Transform.h @@ -22,6 +22,8 @@ #include #include +#include +#include #include @@ -63,7 +65,7 @@ public: uint32_t getType() const; uint32_t getOrientation() const; - float const* operator [] (int i) const; // returns column i + const vec3& operator [] (size_t i) const; // returns column i float tx() const; float ty() const; @@ -75,7 +77,7 @@ public: // transform data Rect makeBounds(int w, int h) const; - void transform(float* point, int x, int y) const; + vec2 transform(int x, int y) const; Region transform(const Region& reg) const; Rect transform(const Rect& bounds) const; Transform operator * (const Transform& rhs) const; @@ -86,24 +88,6 @@ public: void dump(const char* name) const; private: - struct vec3 { - float v[3]; - inline vec3() { } - inline vec3(float a, float b, float c) { - v[0] = a; v[1] = b; v[2] = c; - } - inline float operator [] (int i) const { return v[i]; } - inline float& operator [] (int i) { return v[i]; } - }; - struct vec2 { - float v[2]; - inline vec2() { } - inline vec2(float a, float b) { - v[0] = a; v[1] = b; - } - inline float operator [] (int i) const { return v[i]; } - inline float& operator [] (int i) { return v[i]; } - }; struct mat33 { vec3 v[3]; inline const vec3& operator [] (int i) const { return v[i]; }