camera: Report rotation needed to account for device orientation.

Fixes #11476.

Author: icculus

include/SDL3/SDL_surface.h

@@ -241,6 +241,12 @@ extern SDL_DECLSPEC void SDLCALL SDL_DestroySurface(SDL_Surface *surface);
  *   left edge of the image, if this surface is being used as a cursor.
  * - `SDL_PROP_SURFACE_HOTSPOT_Y_NUMBER`: the hotspot pixel offset from the
  *   top edge of the image, if this surface is being used as a cursor.
+ * - `SDL_PROP_SURFACE_ROTATION_NUMBER`: the number of degrees a surface's
+ *   data is meant to be rotated clockwise to make the image
+ *   right-side up. Default 0. This is used by the camera API, if a mobile
+ *   device is oriented differently than what its camera provides (i.e. -
+ *   the camera always provides portrait images but the phone is being held
+ *   in landscape orientation). Since SDL 3.4.0.
  *
  * \param surface the SDL_Surface structure to query.
  * \returns a valid property ID on success or 0 on failure; call
@@ -257,6 +263,7 @@ extern SDL_DECLSPEC SDL_PropertiesID SDLCALL SDL_GetSurfaceProperties(SDL_Surfac
 #define SDL_PROP_SURFACE_TONEMAP_OPERATOR_STRING            "SDL.surface.tonemap"
 #define SDL_PROP_SURFACE_HOTSPOT_X_NUMBER                   "SDL.surface.hotspot.x"
 #define SDL_PROP_SURFACE_HOTSPOT_Y_NUMBER                   "SDL.surface.hotspot.y"
+#define SDL_PROP_SURFACE_ROTATION_NUMBER                    "SDL.surface.rotation"
 
 /**
  * Set the colorspace used by a surface.

src/camera/SDL_camera.c

@@ -150,7 +150,7 @@ static size_t GetFrameBufLen(const SDL_CameraSpec *spec)
     return wxh * SDL_BYTESPERPIXEL(fmt);
 }
 
-static SDL_CameraFrameResult ZombieAcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult ZombieAcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     const SDL_CameraSpec *spec = &device->actual_spec;
 
@@ -832,9 +832,10 @@ bool SDL_CameraThreadIterate(SDL_Camera *device)
     SDL_Surface *output_surface = NULL;
     SurfaceList *slist = NULL;
     Uint64 timestampNS = 0;
+    int rotation = 0;
 
     // AcquireFrame SHOULD NOT BLOCK, as we are holding a lock right now. Block in WaitDevice instead!
-    const SDL_CameraFrameResult rc = device->AcquireFrame(device, device->acquire_surface, &timestampNS);
+    const SDL_CameraFrameResult rc = device->AcquireFrame(device, device->acquire_surface, &timestampNS, &rotation);
 
     if (rc == SDL_CAMERA_FRAME_READY) {  // new frame acquired!
         #if DEBUG_CAMERA
@@ -928,6 +929,8 @@ bool SDL_CameraThreadIterate(SDL_Camera *device)
         acquired->pixels = NULL;
         acquired->pitch = 0;
 
+        SDL_SetNumberProperty(SDL_GetSurfaceProperties(output_surface), SDL_PROP_SURFACE_ROTATION_NUMBER, rotation);
+
         // make the filled output surface available to the app.
         SDL_LockMutex(device->lock);
         slist->next = device->filled_output_surfaces.next;

src/camera/SDL_syscamera.h

@@ -99,7 +99,7 @@ struct SDL_Camera
 
     // These are, initially, set from camera_driver, but we might swap them out with Zombie versions on disconnect/failure.
     bool (*WaitDevice)(SDL_Camera *device);
-    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS);
+    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation);
     void (*ReleaseFrame)(SDL_Camera *device, SDL_Surface *frame);
 
     // All supported formats/dimensions for this device.
@@ -167,13 +167,18 @@ struct SDL_Camera
     struct SDL_PrivateCameraData *hidden;
 };
 
+
+// Note that for AcquireFrame, `rotation` is degrees, with positive values rotating clockwise. This is the amount to rotate an image so it would be right-side up.
+// Rotations should be in 90 degree increments at this time (landscape to portrait, or upside down to right side up, etc).
+// Most platforms won't care about this, but mobile devices might need to deal with the device itself being physically rotated, causing the fixed-orientation camera to be presenting sideways images.
+
 typedef struct SDL_CameraDriverImpl
 {
     void (*DetectDevices)(void);
     bool (*OpenDevice)(SDL_Camera *device, const SDL_CameraSpec *spec);
     void (*CloseDevice)(SDL_Camera *device);
     bool (*WaitDevice)(SDL_Camera *device);
-    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS); // set frame->pixels, frame->pitch, and *timestampNS!
+    SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation); // set frame->pixels, frame->pitch, *timestampNS, and *rotation!
     void (*ReleaseFrame)(SDL_Camera *device, SDL_Surface *frame); // Reclaim frame->pixels and frame->pitch!
     void (*FreeDeviceHandle)(SDL_Camera *device); // SDL is done with this device; free the handle from SDL_AddCamera()
     void (*Deinitialize)(void);

src/camera/android/SDL_camera_android.c

@@ -295,7 +295,7 @@ static bool ANDROIDCAMERA_WaitDevice(SDL_Camera *device)
     return true;  // this isn't used atm, since we run our own thread via onImageAvailable callbacks.
 }
 
-static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_CameraFrameResult result = SDL_CAMERA_FRAME_READY;
     media_status_t res;

src/camera/coremedia/SDL_camera_coremedia.m

@@ -29,6 +29,10 @@
 #import <AVFoundation/AVFoundation.h>
 #import <CoreMedia/CoreMedia.h>
 
+#if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS)
+#import <UIKit/UIKit.h>
+#endif
+
 /*
  * Need to link with:: CoreMedia CoreVideo
  *
@@ -77,6 +81,9 @@ @interface SDLPrivateCameraData : NSObject
 @property(nonatomic, retain) AVCaptureSession *session;
 @property(nonatomic, retain) SDLCaptureVideoDataOutputSampleBufferDelegate *delegate;
 @property(nonatomic, assign) CMSampleBufferRef current_sample;
+#if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS)
+@property(nonatomic, assign) UIDeviceOrientation last_device_orientation;
+#endif
 @end
 
 @implementation SDLPrivateCameraData
@@ -146,7 +153,7 @@ static bool COREMEDIA_WaitDevice(SDL_Camera *device)
     return true;  // this isn't used atm, since we run our own thread out of Grand Central Dispatch.
 }
 
-static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_CameraFrameResult result = SDL_CAMERA_FRAME_READY;
     SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden;
@@ -219,6 +226,25 @@ static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surf
 
     CVPixelBufferUnlockBaseAddress(image, 0);
 
+    // As of this writing, all known iOS devices provide portrait orientation camera data, so we just need to know how to rotate between that and the current device orientation.
+    // macOS currently assumes you don't ever rotate images because the camera is always positioned right-side up (for now).
+    #if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS)
+    UIDeviceOrientation device_orientation = [[UIDevice currentDevice] orientation];
+    if (!UIDeviceOrientationIsValidInterfaceOrientation(device_orientation)) {
+        device_orientation = hidden.last_device_orientation;  // possible the phone is laying flat or something went wrong, just stay with the last known-good orientation.
+    } else {
+        hidden.last_device_orientation = device_orientation;  // update the last known-good orientation for later.
+    }
+
+    switch (device_orientation) {
+        case UIDeviceOrientationPortrait: *rotation = 0; break;
+        case UIDeviceOrientationPortraitUpsideDown: *rotation = 180; break;
+        case UIDeviceOrientationLandscapeRight: *rotation = 90; break;  // !!! FIXME: might be backwards.
+        case UIDeviceOrientationLandscapeLeft: *rotation = 270; break;  // !!! FIXME: might be backwards.
+        default: SDL_assert(!"Unexpected device orientation!"); *rotation = 0; break;
+    }
+    #endif
+
     return result;
 }
 
@@ -231,6 +257,10 @@ static void COREMEDIA_ReleaseFrame(SDL_Camera *device, SDL_Surface *frame)
 static void COREMEDIA_CloseDevice(SDL_Camera *device)
 {
     if (device && device->hidden) {
+        #if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS)
+        [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications];
+        #endif
+
         SDLPrivateCameraData *hidden = (SDLPrivateCameraData *) CFBridgingRelease(device->hidden);
         device->hidden = NULL;
 
@@ -358,6 +388,28 @@ static bool COREMEDIA_OpenDevice(SDL_Camera *device, const SDL_CameraSpec *spec)
     hidden.session = session;
     hidden.delegate = delegate;
     hidden.current_sample = NULL;
+
+    #if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS)
+    // When using a camera, we turn on device orientation tracking. The docs note that this turns on
+    // the device's accelerometer, so I assume this burns power, so we don't leave this running all
+    // the time. These calls nest, so we just need to call the matching `end` message when we close.
+    // You _can_ get an actual events through this mechanism, but we just want to be able to call
+    // -[UIDevice orientation], which will update with real info while notificatons are enabled.
+    UIDevice *uidevice = [UIDevice currentDevice];
+    [uidevice beginGeneratingDeviceOrientationNotifications];
+    hidden.last_device_orientation = uidevice.orientation;
+    if (!UIDeviceOrientationIsValidInterfaceOrientation(hidden.last_device_orientation)) {
+        // accelerometer isn't ready yet or the phone is laying flat or something. Just try to guess from how the UI is oriented at the moment.
+        switch ([UIApplication sharedApplication].statusBarOrientation) {
+            case UIInterfaceOrientationPortrait: hidden.last_device_orientation = UIDeviceOrientationPortrait; break;
+            case UIInterfaceOrientationPortraitUpsideDown: hidden.last_device_orientation = UIDeviceOrientationPortraitUpsideDown; break;
+            case UIInterfaceOrientationLandscapeLeft: hidden.last_device_orientation = UIDeviceOrientationLandscapeRight; break;  // Apple docs say UI and device orientations are reversed in landscape.
+            case UIInterfaceOrientationLandscapeRight: hidden.last_device_orientation = UIDeviceOrientationLandscapeLeft; break;
+            default: hidden.last_device_orientation = UIDeviceOrientationPortrait; break;  // oh well.
+        }
+    }
+    #endif
+
     device->hidden = (struct SDL_PrivateCameraData *)CFBridgingRetain(hidden);
 
     [session startRunning];  // !!! FIXME: docs say this can block while camera warms up and shouldn't be done on main thread. Maybe push through `queue`?

src/camera/dummy/SDL_camera_dummy.c

@@ -38,7 +38,7 @@ static bool DUMMYCAMERA_WaitDevice(SDL_Camera *device)
     return SDL_Unsupported();
 }
 
-static SDL_CameraFrameResult DUMMYCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult DUMMYCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_Unsupported();
     return SDL_CAMERA_FRAME_ERROR;

src/camera/emscripten/SDL_camera_emscripten.c

@@ -39,7 +39,7 @@ static bool EMSCRIPTENCAMERA_WaitDevice(SDL_Camera *device)
     return false;
 }
 
-static SDL_CameraFrameResult EMSCRIPTENCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult EMSCRIPTENCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     void *rgba = SDL_malloc(device->actual_spec.width * device->actual_spec.height * 4);
     if (!rgba) {

src/camera/mediafoundation/SDL_camera_mediafoundation.c

@@ -430,7 +430,7 @@ static void SDLCALL CleanupIMFMediaBuffer(void *userdata, void *value)
     SDL_free(objs);
 }
 
-static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_assert(device->hidden->current_sample != NULL);
 
@@ -562,7 +562,7 @@ static SDL_CameraFrameResult MEDIAFOUNDATION_CopyFrame(SDL_Surface *frame, const
     return SDL_CAMERA_FRAME_READY;
 }
 
-static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     SDL_assert(device->hidden->current_sample != NULL);
 

src/camera/pipewire/SDL_camera_pipewire.c

@@ -577,7 +577,7 @@ static bool PIPEWIRECAMERA_WaitDevice(SDL_Camera *device)
     return true;
 }
 
-static SDL_CameraFrameResult PIPEWIRECAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult PIPEWIRECAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     struct pw_buffer *b;
 

src/camera/v4l2/SDL_camera_v4l2.c

@@ -125,7 +125,7 @@ static bool V4L2_WaitDevice(SDL_Camera *device)
     return false;
 }
 
-static SDL_CameraFrameResult V4L2_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS)
+static SDL_CameraFrameResult V4L2_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation)
 {
     const int fd = device->hidden->fd;
     const io_method io = device->hidden->io;