fix camera API 2.0 orientation

we add a flag to ANativeWindow::setBufferTransform that means
"apply the inverse rotation of the display this buffer is displayed
onto to".

Bug: 10804238
Change-Id: Id2447676271950463e8dbcef1b95935c5c3f32b2

include/gui/IGraphicBufferConsumer.h

@@ -91,6 +91,10 @@ public:
 
         // Indicates whether this buffer has been seen by a consumer yet
         bool mAcquireCalled;
+
+        // Indicates this buffer must be transformed by the inverse transform of the screen
+        // it is displayed onto. This is applied after mTransform.
+        bool mTransformToDisplayInverse;
     };
 
 

include/ui/TMatHelpers.h

@@ -245,7 +245,7 @@ template <template<typename T> class BASE, typename T>
 class TMatDebug {
 public:
     String8 asString() const {
-        return matrix::asString(*this);
+        return matrix::asString( static_cast< const BASE<T>& >(*this) );
     }
 };
 

libs/gui/BufferQueue.cpp

@@ -557,7 +557,8 @@ status_t BufferQueue::queueBuffer(int buf,
         item.mAcquireCalled = mSlots[buf].mAcquireCalled;
         item.mGraphicBuffer = mSlots[buf].mGraphicBuffer;
         item.mCrop = crop;
-        item.mTransform = transform;
+        item.mTransform = transform & ~NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY;
+        item.mTransformToDisplayInverse = bool(transform & NATIVE_WINDOW_TRANSFORM_INVERSE_DISPLAY);
         item.mScalingMode = scalingMode;
         item.mTimestamp = timestamp;
         item.mIsAutoTimestamp = isAutoTimestamp;

libs/gui/IGraphicBufferConsumer.cpp

@@ -47,7 +47,8 @@ IGraphicBufferConsumer::BufferItem::BufferItem() :
     mFrameNumber(0),
     mBuf(INVALID_BUFFER_SLOT),
     mIsDroppable(false),
-    mAcquireCalled(false) {
+    mAcquireCalled(false),
+    mTransformToDisplayInverse(false) {
     mCrop.makeInvalid();
 }
 
@@ -60,7 +61,8 @@ size_t IGraphicBufferConsumer::BufferItem::getPodSize() const {
             sizeof(mFrameNumber) +
             sizeof(mBuf) +
             sizeof(mIsDroppable) +
-            sizeof(mAcquireCalled);
+            sizeof(mAcquireCalled) +
+            sizeof(mTransformToDisplayInverse);
     return c;
 }
 
@@ -130,6 +132,7 @@ status_t IGraphicBufferConsumer::BufferItem::flatten(
     FlattenableUtils::write(buffer, size, mBuf);
     FlattenableUtils::write(buffer, size, mIsDroppable);
     FlattenableUtils::write(buffer, size, mAcquireCalled);
+    FlattenableUtils::write(buffer, size, mTransformToDisplayInverse);
 
     return NO_ERROR;
 }
@@ -171,6 +174,7 @@ status_t IGraphicBufferConsumer::BufferItem::unflatten(
     FlattenableUtils::read(buffer, size, mBuf);
     FlattenableUtils::read(buffer, size, mIsDroppable);
     FlattenableUtils::read(buffer, size, mAcquireCalled);
+    FlattenableUtils::read(buffer, size, mTransformToDisplayInverse);
 
     return NO_ERROR;
 }

services/surfaceflinger/DisplayDevice.cpp

@@ -344,6 +344,25 @@ void DisplayDevice::setLayerStack(uint32_t stack) {
 
 // ----------------------------------------------------------------------------
 
+uint32_t DisplayDevice::getOrientationTransform() const {
+    uint32_t transform = 0;
+    switch (mOrientation) {
+        case DisplayState::eOrientationDefault:
+            transform = Transform::ROT_0;
+            break;
+        case DisplayState::eOrientation90:
+            transform = Transform::ROT_90;
+            break;
+        case DisplayState::eOrientation180:
+            transform = Transform::ROT_180;
+            break;
+        case DisplayState::eOrientation270:
+            transform = Transform::ROT_270;
+            break;
+    }
+    return transform;
+}
+
 status_t DisplayDevice::orientationToTransfrom(
         int orientation, int w, int h, Transform* tr)
 {

services/surfaceflinger/DisplayDevice.h

@@ -111,6 +111,7 @@ public:
     void                    setProjection(int orientation, const Rect& viewport, const Rect& frame);
 
     int                     getOrientation() const { return mOrientation; }
+    uint32_t                getOrientationTransform() const;
     const Transform&        getTransform() const { return mGlobalTransform; }
     const Rect              getViewport() const { return mViewport; }
     const Rect              getFrame() const { return mFrame; }

services/surfaceflinger/Layer.cpp

@@ -376,7 +376,21 @@ void Layer::setGeometry(
      */
 
     const Transform bufferOrientation(mCurrentTransform);
-    const Transform transform(tr * s.transform * bufferOrientation);
+    Transform transform(tr * s.transform * bufferOrientation);
+
+    if (mSurfaceFlingerConsumer->getTransformToDisplayInverse()) {
+        /*
+         * the code below applies the display's inverse transform to the buffer
+         */
+        uint32_t invTransform = hw->getOrientationTransform();
+        // calculate the inverse transform
+        if (invTransform & NATIVE_WINDOW_TRANSFORM_ROT_90) {
+            invTransform ^= NATIVE_WINDOW_TRANSFORM_FLIP_V |
+                    NATIVE_WINDOW_TRANSFORM_FLIP_H;
+        }
+        // and apply to the current transform
+        transform = transform * Transform(invTransform);
+    }
 
     // this gives us only the "orientation" component of the transform
     const uint32_t orientation = transform.getOrientation();
@@ -489,6 +503,34 @@ void Layer::onDraw(const sp<const DisplayDevice>& hw, const Region& clip) const
         mSurfaceFlingerConsumer->setFilteringEnabled(useFiltering);
         mSurfaceFlingerConsumer->getTransformMatrix(textureMatrix);
 
+        if (mSurfaceFlingerConsumer->getTransformToDisplayInverse()) {
+
+            /*
+             * the code below applies the display's inverse transform to the texture transform
+             */
+
+            // create a 4x4 transform matrix from the display transform flags
+            const mat4 flipH(-1,0,0,0,  0,1,0,0, 0,0,1,0, 1,0,0,1);
+            const mat4 flipV( 1,0,0,0, 0,-1,0,0, 0,0,1,0, 0,1,0,1);
+            const mat4 rot90( 0,1,0,0, -1,0,0,0, 0,0,1,0, 1,0,0,1);
+
+            mat4 tr;
+            uint32_t transform = hw->getOrientationTransform();
+            if (transform & NATIVE_WINDOW_TRANSFORM_ROT_90)
+                tr = tr * rot90;
+            if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_H)
+                tr = tr * flipH;
+            if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_V)
+                tr = tr * flipV;
+
+            // calculate the inverse
+            tr = inverse(tr);
+
+            // and finally apply it to the original texture matrix
+            const mat4 texTransform(mat4(static_cast<const float*>(textureMatrix)) * tr);
+            memcpy(textureMatrix, texTransform.asArray(), sizeof(textureMatrix));
+        }
+
         // Set things up for texturing.
         mTexture.setDimensions(mActiveBuffer->getWidth(), mActiveBuffer->getHeight());
         mTexture.setFiltering(useFiltering);
@@ -533,7 +575,7 @@ void Layer::drawWithOpenGL(
      *
      * The GL code below is more logical (imho), and the difference with
      * HWC is due to a limitation of the HWC API to integers -- a question
-     * is suspend is wether we should ignore this problem or revert to
+     * is suspend is whether we should ignore this problem or revert to
      * GL composition when a buffer scaling is applied (maybe with some
      * minimal value)? Or, we could make GL behave like HWC -- but this feel
      * like more of a hack.

services/surfaceflinger/SurfaceFlingerConsumer.cpp

@@ -99,6 +99,19 @@ status_t SurfaceFlingerConsumer::bindTextureImage()
     return bindTextureImageLocked();
 }
 
+status_t SurfaceFlingerConsumer::acquireBufferLocked(
+        BufferQueue::BufferItem *item, nsecs_t presentWhen) {
+    status_t result = GLConsumer::acquireBufferLocked(item, presentWhen);
+    if (result == NO_ERROR) {
+        mTransformToDisplayInverse = item->mTransformToDisplayInverse;
+    }
+    return result;
+}
+
+bool SurfaceFlingerConsumer::getTransformToDisplayInverse() const {
+    return mTransformToDisplayInverse;
+}
+
 // We need to determine the time when a buffer acquired now will be
 // displayed.  This can be calculated:
 //   time when previous buffer's actual-present fence was signaled

services/surfaceflinger/SurfaceFlingerConsumer.h

@@ -40,6 +40,8 @@ public:
         virtual ~BufferRejecter() { }
     };
 
+    virtual status_t acquireBufferLocked(BufferQueue::BufferItem *item, nsecs_t presentWhen);
+
     // This version of updateTexImage() takes a functor that may be used to
     // reject the newly acquired buffer.  Unlike the GLConsumer version,
     // this does not guarantee that the buffer has been bound to the GL
@@ -49,8 +51,16 @@ public:
     // See GLConsumer::bindTextureImageLocked().
     status_t bindTextureImage();
 
+    // must be called from SF main thread
+    bool getTransformToDisplayInverse() const;
+
 private:
     nsecs_t computeExpectedPresent();
+
+    // Indicates this buffer must be transformed by the inverse transform of the screen
+    // it is displayed onto. This is applied after GLConsumer::mCurrentTransform.
+    // This must be set/read from SurfaceFlinger's main thread.
+    bool mTransformToDisplayInverse;
 };
 
 // ----------------------------------------------------------------------------