Properly adjust the orientation for full-screen shots (#16)

* Properly adjust the orientation for full-screen shots

* Only fix the orientation if needed

* Revert "Only fix the orientation if needed"

This reverts commit fae7914d5df7752ef131af2845e4bfc2131021ed.

* Tune error messages

* Fix taking screenshots for elements with 'correct' frames in landscape mode

* Apply some small refactoring

* Use appFrame everywhere

Author: Mykola Mokhnach

WebDriverAgentLib/Categories/XCUIDevice+FBHelpers.m

@@ -56,7 +56,8 @@ - (NSData *)fb_screenshotWithError:(NSError*__autoreleasing*)error
   }
 
   id mainScreen = [xcScreen valueForKey:@"mainScreen"];
-  CGSize screenSize = FBAdjustDimensionsForApplication(FBApplication.fb_activeApplication.frame.size, (UIInterfaceOrientation)[self.class sharedDevice].orientation);
+  FBApplication *activeApplication = FBApplication.fb_activeApplication;
+  UIInterfaceOrientation orientation = activeApplication.interfaceOrientation;
   SEL mSelector = NSSelectorFromString(@"screenshotDataForQuality:rect:error:");
   NSMethodSignature *mSignature = [mainScreen methodSignatureForSelector:mSelector];
   NSInvocation *invocation = [NSInvocation invocationWithMethodSignature:mSignature];
@@ -67,18 +68,17 @@ - (NSData *)fb_screenshotWithError:(NSError*__autoreleasing*)error
   // and the resulting screenshot does not fit the memory buffer preallocated for it by the operating system
   NSUInteger quality = 1;
   [invocation setArgument:&quality atIndex:2];
+  CGSize screenSize = FBAdjustDimensionsForApplication(activeApplication.frame.size, orientation);
   CGRect screenRect = CGRectMake(0, 0, screenSize.width, screenSize.height);
   [invocation setArgument:&screenRect atIndex:3];
   [invocation setArgument:&error atIndex:4];
   [invocation invoke];
-  NSData __unsafe_unretained *result;
-  [invocation getReturnValue:&result];
-  if (nil == result) {
+  NSData __unsafe_unretained *imageData;
+  [invocation getReturnValue:&imageData];
+  if (nil == imageData) {
     return nil;
   }
-  // The resulting data is a JPEG image, so we need to convert it to PNG representation
-  UIImage *image = [UIImage imageWithData:result];
-  return (NSData *)UIImagePNGRepresentation(image);
+  return FBAdjustScreenshotOrientationForApplication(imageData, orientation);
 }
 
 - (BOOL)fb_fingerTouchShouldMatch:(BOOL)shouldMatch

WebDriverAgentLib/Categories/XCUIElement+FBUtilities.m

@@ -153,6 +153,18 @@ - (NSData *)fb_screenshotWithError:(NSError **)error
   NSUInteger quality = 1;
   [invocation setArgument:&quality atIndex:2];
   CGRect elementRect = self.frame;
+  UIInterfaceOrientation orientation = self.application.interfaceOrientation;
+  if (orientation == UIInterfaceOrientationLandscapeLeft || orientation == UIInterfaceOrientationLandscapeRight) {
+    // Workaround XCTest bug when element frame is returned as in portrait mode even if the screenshot is rotated
+    XCElementSnapshot *parentWindow = [self.fb_lastSnapshot fb_parentMatchingType:XCUIElementTypeWindow];
+    CGRect appFrame = self.application.frame;
+    if (CGRectEqualToRect(appFrame, nil == parentWindow ? elementRect : parentWindow.frame)) {
+      CGPoint fixedOrigin = orientation == UIInterfaceOrientationLandscapeLeft ?
+        CGPointMake(appFrame.size.height - elementRect.origin.y - elementRect.size.height, elementRect.origin.x) :
+        CGPointMake(elementRect.origin.y, appFrame.size.width - elementRect.origin.x - elementRect.size.width);
+      elementRect = CGRectMake(fixedOrigin.x, fixedOrigin.y, elementRect.size.height, elementRect.size.width);
+    }
+  }
   [invocation setArgument:&elementRect atIndex:3];
   [invocation setArgument:&error atIndex:4];
   [invocation invoke];
@@ -161,26 +173,7 @@ - (NSData *)fb_screenshotWithError:(NSError **)error
   if (nil == imageData) {
     return nil;
   }
-
-  UIImage *image = [UIImage imageWithData:imageData];
-  UIInterfaceOrientation orientation = self.application.interfaceOrientation;
-  UIImageOrientation imageOrientation = UIImageOrientationUp;
-  // The received element screenshot will be rotated, if the current interface orientation differs from portrait, so we need to fix that first
-  if (orientation == UIInterfaceOrientationLandscapeRight) {
-    imageOrientation = UIImageOrientationLeft;
-  } else if (orientation == UIInterfaceOrientationLandscapeLeft) {
-    imageOrientation = UIImageOrientationRight;
-  } else if (orientation == UIInterfaceOrientationPortraitUpsideDown) {
-    imageOrientation = UIImageOrientationDown;
-  }
-  CGSize size = image.size;
-  UIGraphicsBeginImageContext(CGSizeMake(size.width, size.height));
-  [[UIImage imageWithCGImage:(CGImageRef)[image CGImage] scale:1.0 orientation:imageOrientation] drawInRect:CGRectMake(0, 0, size.width, size.height)];
-  UIImage *fixedImage = UIGraphicsGetImageFromCurrentImageContext();
-  UIGraphicsEndImageContext();
-
-  // The resulting data is a JPEG image, so we need to convert it to PNG representation
-  return (NSData *)UIImagePNGRepresentation(fixedImage);
+  return FBAdjustScreenshotOrientationForApplication(imageData, orientation);
 }
 
 @end

WebDriverAgentLib/Utilities/FBBaseActionsSynthesizer.m

@@ -52,7 +52,7 @@ - (nullable NSValue *)hitpointWithElement:(nullable XCUIElement *)element positi
     XCElementSnapshot *snapshot = element.fb_lastSnapshot;
     CGRect frameInWindow = snapshot.fb_frameInWindow;
     if (CGRectIsEmpty(frameInWindow)) {
-      NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen", element.debugDescription];
+      NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description];
       if (error) {
         *error = [[FBErrorBuilder.builder withDescription:description] build];
       }

WebDriverAgentLib/Utilities/FBMathUtils.h

@@ -36,3 +36,6 @@ CGPoint FBInvertOffsetForOrientation(CGPoint offset, UIInterfaceOrientation orie
 
 /*! Inverts size if necessary to match current screen orientation */
 CGSize FBAdjustDimensionsForApplication(CGSize actualSize, UIInterfaceOrientation orientation);
+
+/*! Fixes the screenshot orientation if necessary to match current screen orientation */
+NSData *FBAdjustScreenshotOrientationForApplication(NSData *screenshotData, UIInterfaceOrientation orientation);

WebDriverAgentLib/Utilities/FBMathUtils.m

@@ -82,3 +82,27 @@ This verification is just to make sure the bug is still there (since height is n
   }
   return actualSize;
 }
+
+NSData *FBAdjustScreenshotOrientationForApplication(NSData *screenshotData, UIInterfaceOrientation orientation)
+{
+  UIImage *image = [UIImage imageWithData:screenshotData];
+  UIImageOrientation imageOrientation;
+  if (orientation == UIInterfaceOrientationLandscapeRight) {
+    imageOrientation = UIImageOrientationLeft;
+  } else if (orientation == UIInterfaceOrientationLandscapeLeft) {
+    imageOrientation = UIImageOrientationRight;
+  } else if (orientation == UIInterfaceOrientationPortraitUpsideDown) {
+    imageOrientation = UIImageOrientationDown;
+  } else {
+    return (NSData *)UIImagePNGRepresentation(image);
+  }
+  
+  UIGraphicsBeginImageContext(CGSizeMake(image.size.width, image.size.height));
+  [[UIImage imageWithCGImage:(CGImageRef)[image CGImage] scale:1.0 orientation:imageOrientation]
+   drawInRect:CGRectMake(0, 0, image.size.width, image.size.height)];
+  UIImage *fixedImage = UIGraphicsGetImageFromCurrentImageContext();
+  UIGraphicsEndImageContext();
+  
+  // The resulting data should be a PNG image
+  return (NSData *)UIImagePNGRepresentation(fixedImage);
+}

WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m

@@ -131,7 +131,7 @@ - (nullable NSValue *)hitpointWithElement:(nullable XCUIElement *)element positi
     // An offset relative to the element is defined
     XCElementSnapshot *snapshot = element.fb_lastSnapshot;
     if (CGRectIsEmpty(snapshot.fb_frameInWindow)) {
-      NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen", element.debugDescription];
+      NSString *description = [NSString stringWithFormat:@"The element '%@' is not visible on the screen and thus is not interactable", element.description];
       if (error) {
         *error = [[FBErrorBuilder.builder withDescription:description] build];
       }