From c10af8e354433674d13b6ab633b15e145e7c00d0 Mon Sep 17 00:00:00 2001 From: "Ryan C. Gordon" Date: Tue, 2 Dec 2025 20:26:08 -0500 Subject: [PATCH] camera: Report rotation needed to account for device orientation. Fixes #11476. --- include/SDL3/SDL_surface.h | 7 ++ src/camera/SDL_camera.c | 7 +- src/camera/SDL_syscamera.h | 9 ++- src/camera/android/SDL_camera_android.c | 33 ++++++++- src/camera/coremedia/SDL_camera_coremedia.m | 70 ++++++++++++++++++- src/camera/dummy/SDL_camera_dummy.c | 2 +- src/camera/emscripten/SDL_camera_emscripten.c | 2 +- .../SDL_camera_mediafoundation.c | 4 +- src/camera/pipewire/SDL_camera_pipewire.c | 2 +- src/camera/v4l2/SDL_camera_v4l2.c | 2 +- src/camera/vita/SDL_camera_vita.c | 2 +- test/testcamera.c | 7 +- 12 files changed, 132 insertions(+), 15 deletions(-) diff --git a/include/SDL3/SDL_surface.h b/include/SDL3/SDL_surface.h index 053557fc2fd05..b46491d50d552 100644 --- a/include/SDL3/SDL_surface.h +++ b/include/SDL3/SDL_surface.h @@ -241,6 +241,12 @@ extern SDL_DECLSPEC void SDLCALL SDL_DestroySurface(SDL_Surface *surface); * left edge of the image, if this surface is being used as a cursor. * - `SDL_PROP_SURFACE_HOTSPOT_Y_NUMBER`: the hotspot pixel offset from the * top edge of the image, if this surface is being used as a cursor. + * - `SDL_PROP_SURFACE_ROTATION_NUMBER`: the number of degrees a surface's + * data is meant to be rotated clockwise to make the image + * right-side up. Default 0. This is used by the camera API, if a mobile + * device is oriented differently than what its camera provides (i.e. - + * the camera always provides portrait images but the phone is being held + * in landscape orientation). Since SDL 3.4.0. * * \param surface the SDL_Surface structure to query. * \returns a valid property ID on success or 0 on failure; call @@ -257,6 +263,7 @@ extern SDL_DECLSPEC SDL_PropertiesID SDLCALL SDL_GetSurfaceProperties(SDL_Surfac #define SDL_PROP_SURFACE_TONEMAP_OPERATOR_STRING "SDL.surface.tonemap" #define SDL_PROP_SURFACE_HOTSPOT_X_NUMBER "SDL.surface.hotspot.x" #define SDL_PROP_SURFACE_HOTSPOT_Y_NUMBER "SDL.surface.hotspot.y" +#define SDL_PROP_SURFACE_ROTATION_NUMBER "SDL.surface.rotation" /** * Set the colorspace used by a surface. diff --git a/src/camera/SDL_camera.c b/src/camera/SDL_camera.c index a8f9e8856ec3a..b3c3e5e82cc7b 100644 --- a/src/camera/SDL_camera.c +++ b/src/camera/SDL_camera.c @@ -150,7 +150,7 @@ static size_t GetFrameBufLen(const SDL_CameraSpec *spec) return wxh * SDL_BYTESPERPIXEL(fmt); } -static SDL_CameraFrameResult ZombieAcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult ZombieAcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { const SDL_CameraSpec *spec = &device->actual_spec; @@ -832,9 +832,10 @@ bool SDL_CameraThreadIterate(SDL_Camera *device) SDL_Surface *output_surface = NULL; SurfaceList *slist = NULL; Uint64 timestampNS = 0; + int rotation = 0; // AcquireFrame SHOULD NOT BLOCK, as we are holding a lock right now. Block in WaitDevice instead! - const SDL_CameraFrameResult rc = device->AcquireFrame(device, device->acquire_surface, ×tampNS); + const SDL_CameraFrameResult rc = device->AcquireFrame(device, device->acquire_surface, ×tampNS, &rotation); if (rc == SDL_CAMERA_FRAME_READY) { // new frame acquired! #if DEBUG_CAMERA @@ -928,6 +929,8 @@ bool SDL_CameraThreadIterate(SDL_Camera *device) acquired->pixels = NULL; acquired->pitch = 0; + SDL_SetNumberProperty(SDL_GetSurfaceProperties(output_surface), SDL_PROP_SURFACE_ROTATION_NUMBER, rotation); + // make the filled output surface available to the app. SDL_LockMutex(device->lock); slist->next = device->filled_output_surfaces.next; diff --git a/src/camera/SDL_syscamera.h b/src/camera/SDL_syscamera.h index 094ad0faa99d1..151f73d65a0e7 100644 --- a/src/camera/SDL_syscamera.h +++ b/src/camera/SDL_syscamera.h @@ -99,7 +99,7 @@ struct SDL_Camera // These are, initially, set from camera_driver, but we might swap them out with Zombie versions on disconnect/failure. bool (*WaitDevice)(SDL_Camera *device); - SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS); + SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation); void (*ReleaseFrame)(SDL_Camera *device, SDL_Surface *frame); // All supported formats/dimensions for this device. @@ -167,13 +167,18 @@ struct SDL_Camera struct SDL_PrivateCameraData *hidden; }; + +// Note that for AcquireFrame, `rotation` is degrees, with positive values rotating clockwise. This is the amount to rotate an image so it would be right-side up. +// Rotations should be in 90 degree increments at this time (landscape to portrait, or upside down to right side up, etc). +// Most platforms won't care about this, but mobile devices might need to deal with the device itself being physically rotated, causing the fixed-orientation camera to be presenting sideways images. + typedef struct SDL_CameraDriverImpl { void (*DetectDevices)(void); bool (*OpenDevice)(SDL_Camera *device, const SDL_CameraSpec *spec); void (*CloseDevice)(SDL_Camera *device); bool (*WaitDevice)(SDL_Camera *device); - SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS); // set frame->pixels, frame->pitch, and *timestampNS! + SDL_CameraFrameResult (*AcquireFrame)(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation); // set frame->pixels, frame->pitch, *timestampNS, and *rotation! void (*ReleaseFrame)(SDL_Camera *device, SDL_Surface *frame); // Reclaim frame->pixels and frame->pitch! void (*FreeDeviceHandle)(SDL_Camera *device); // SDL is done with this device; free the handle from SDL_AddCamera() void (*Deinitialize)(void); diff --git a/src/camera/android/SDL_camera_android.c b/src/camera/android/SDL_camera_android.c index 90eb0007c5421..e52c78f3a747b 100644 --- a/src/camera/android/SDL_camera_android.c +++ b/src/camera/android/SDL_camera_android.c @@ -149,6 +149,7 @@ struct SDL_PrivateCameraData ACaptureRequest *request; ACameraCaptureSession *session; SDL_CameraSpec requested_spec; + int rotation; // degrees to rotate clockwise to get from camera's static orientation to device's native orientation. Apply this plus current phone rotation to get upright image! }; static bool SetErrorStr(const char *what, const char *errstr, const int rc) @@ -295,7 +296,7 @@ static bool ANDROIDCAMERA_WaitDevice(SDL_Camera *device) return true; // this isn't used atm, since we run our own thread via onImageAvailable callbacks. } -static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { SDL_CameraFrameResult result = SDL_CAMERA_FRAME_READY; media_status_t res; @@ -366,6 +367,21 @@ static SDL_CameraFrameResult ANDROIDCAMERA_AcquireFrame(SDL_Camera *device, SDL_ pAImage_delete(image); + int dev_rotation = 0; + switch (Android_JNI_GetDisplayCurrentOrientation()) { + case SDL_ORIENTATION_PORTRAIT: dev_rotation = 0; break; + case SDL_ORIENTATION_LANDSCAPE: dev_rotation = 90; break; + case SDL_ORIENTATION_PORTRAIT_FLIPPED: dev_rotation = 180; break; + case SDL_ORIENTATION_LANDSCAPE_FLIPPED: dev_rotation = 270; break; + default: SDL_assert(!"Unexpected device rotation!"); dev_rotation = 0; break; + } + + if (device->position == SDL_CAMERA_POSITION_BACK_FACING) { + dev_rotation = -dev_rotation; // we want to subtract this value, instead of add, if back-facing. + } + + *rotation = dev_rotation + device->hidden->rotation; // current phone orientation, static camera orientation in relation to phone. + return result; } @@ -494,10 +510,23 @@ static bool PrepareCamera(SDL_Camera *device) imglistener.context = device; imglistener.onImageAvailable = onImageAvailable; + + const char *devid = (const char *) device->handle; + + device->hidden->rotation = 0; + ACameraMetadata *metadata = NULL; + ACameraMetadata_const_entry orientationentry; + if (pACameraManager_getCameraCharacteristics(cameraMgr, devid, &metadata) == ACAMERA_OK) { + if (pACameraMetadata_getConstEntry(metadata, ACAMERA_SENSOR_ORIENTATION, &orientationentry) == ACAMERA_OK) { + device->hidden->rotation = (int) (*orientationentry.data.i32 % 360); + } + pACameraMetadata_free(metadata); + } + // just in case SDL_OpenCamera is overwriting device->spec as CameraPermissionCallback runs, we work from a different copy. const SDL_CameraSpec *spec = &device->hidden->requested_spec; - if ((res = pACameraManager_openCamera(cameraMgr, (const char *) device->handle, &dev_callbacks, &device->hidden->device)) != ACAMERA_OK) { + if ((res = pACameraManager_openCamera(cameraMgr, devid, &dev_callbacks, &device->hidden->device)) != ACAMERA_OK) { return SetCameraError("Failed to open camera", res); } else if ((res2 = pAImageReader_new(spec->width, spec->height, format_sdl_to_android(spec->format), 10 /* nb buffers */, &device->hidden->reader)) != AMEDIA_OK) { return SetMediaError("Error AImageReader_new", res2); diff --git a/src/camera/coremedia/SDL_camera_coremedia.m b/src/camera/coremedia/SDL_camera_coremedia.m index 94b71109704f4..b50ceba953008 100644 --- a/src/camera/coremedia/SDL_camera_coremedia.m +++ b/src/camera/coremedia/SDL_camera_coremedia.m @@ -29,6 +29,14 @@ #import #import +#if defined(SDL_PLATFORM_IOS) && !defined(SDL_PLATFORM_TVOS) +#define USE_UIKIT_DEVICE_ROTATION +#endif + +#ifdef USE_UIKIT_DEVICE_ROTATION +#import +#endif + /* * Need to link with:: CoreMedia CoreVideo * @@ -77,6 +85,9 @@ @interface SDLPrivateCameraData : NSObject @property(nonatomic, retain) AVCaptureSession *session; @property(nonatomic, retain) SDLCaptureVideoDataOutputSampleBufferDelegate *delegate; @property(nonatomic, assign) CMSampleBufferRef current_sample; +#ifdef USE_UIKIT_DEVICE_ROTATION +@property(nonatomic, assign) UIDeviceOrientation last_device_orientation; +#endif @end @implementation SDLPrivateCameraData @@ -146,7 +157,7 @@ static bool COREMEDIA_WaitDevice(SDL_Camera *device) return true; // this isn't used atm, since we run our own thread out of Grand Central Dispatch. } -static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { SDL_CameraFrameResult result = SDL_CAMERA_FRAME_READY; SDLPrivateCameraData *hidden = (__bridge SDLPrivateCameraData *) device->hidden; @@ -219,6 +230,37 @@ static SDL_CameraFrameResult COREMEDIA_AcquireFrame(SDL_Camera *device, SDL_Surf CVPixelBufferUnlockBaseAddress(image, 0); + #ifdef USE_UIKIT_DEVICE_ROTATION + UIDeviceOrientation device_orientation = [[UIDevice currentDevice] orientation]; + if (!UIDeviceOrientationIsValidInterfaceOrientation(device_orientation)) { + device_orientation = hidden.last_device_orientation; // possible the phone is laying flat or something went wrong, just stay with the last known-good orientation. + } else { + hidden.last_device_orientation = device_orientation; // update the last known-good orientation for later. + } + + const UIInterfaceOrientation ui_orientation = [UIApplication sharedApplication].statusBarOrientation; + + // there is probably math for this, but this is easy to slap into a table. + // rotation = rotations[uiorientation-1][devorientation-1]; + if (device->position == SDL_CAMERA_POSITION_BACK_FACING) { + static const int back_rotations[4][4] = { + { 90, 90, 90, 90 }, // ui portrait + { 270, 270, 270, 270 }, // ui portait upside down + { 0, 0, 0, 0 }, // ui landscape left + { 180, 180, 180, 180 } // ui landscape right + }; + *rotation = back_rotations[ui_orientation - 1][device_orientation - 1]; + } else { + static const int front_rotations[4][4] = { + { 90, 90, 270, 270 }, // ui portrait + { 270, 270, 90, 90 }, // ui portait upside down + { 0, 0, 180, 180 }, // ui landscape left + { 180, 180, 0, 0 } // ui landscape right + }; + *rotation = front_rotations[ui_orientation - 1][device_orientation - 1]; + } + #endif + return result; } @@ -231,6 +273,10 @@ static void COREMEDIA_ReleaseFrame(SDL_Camera *device, SDL_Surface *frame) static void COREMEDIA_CloseDevice(SDL_Camera *device) { if (device && device->hidden) { + #ifdef USE_UIKIT_DEVICE_ROTATION + [[UIDevice currentDevice] endGeneratingDeviceOrientationNotifications]; + #endif + SDLPrivateCameraData *hidden = (SDLPrivateCameraData *) CFBridgingRelease(device->hidden); device->hidden = NULL; @@ -358,6 +404,28 @@ static bool COREMEDIA_OpenDevice(SDL_Camera *device, const SDL_CameraSpec *spec) hidden.session = session; hidden.delegate = delegate; hidden.current_sample = NULL; + + #ifdef USE_UIKIT_DEVICE_ROTATION + // When using a camera, we turn on device orientation tracking. The docs note that this turns on + // the device's accelerometer, so I assume this burns power, so we don't leave this running all + // the time. These calls nest, so we just need to call the matching `end` message when we close. + // You _can_ get an actual events through this mechanism, but we just want to be able to call + // -[UIDevice orientation], which will update with real info while notificatons are enabled. + UIDevice *uidevice = [UIDevice currentDevice]; + [uidevice beginGeneratingDeviceOrientationNotifications]; + hidden.last_device_orientation = uidevice.orientation; + if (!UIDeviceOrientationIsValidInterfaceOrientation(hidden.last_device_orientation)) { + // accelerometer isn't ready yet or the phone is laying flat or something. Just try to guess from how the UI is oriented at the moment. + switch ([UIApplication sharedApplication].statusBarOrientation) { + case UIInterfaceOrientationPortrait: hidden.last_device_orientation = UIDeviceOrientationPortrait; break; + case UIInterfaceOrientationPortraitUpsideDown: hidden.last_device_orientation = UIDeviceOrientationPortraitUpsideDown; break; + case UIInterfaceOrientationLandscapeLeft: hidden.last_device_orientation = UIDeviceOrientationLandscapeRight; break; // Apple docs say UI and device orientations are reversed in landscape. + case UIInterfaceOrientationLandscapeRight: hidden.last_device_orientation = UIDeviceOrientationLandscapeLeft; break; + default: hidden.last_device_orientation = UIDeviceOrientationPortrait; break; // oh well. + } + } + #endif + device->hidden = (struct SDL_PrivateCameraData *)CFBridgingRetain(hidden); [session startRunning]; // !!! FIXME: docs say this can block while camera warms up and shouldn't be done on main thread. Maybe push through `queue`? diff --git a/src/camera/dummy/SDL_camera_dummy.c b/src/camera/dummy/SDL_camera_dummy.c index b2a4dc136aa84..981c2cecb8739 100644 --- a/src/camera/dummy/SDL_camera_dummy.c +++ b/src/camera/dummy/SDL_camera_dummy.c @@ -38,7 +38,7 @@ static bool DUMMYCAMERA_WaitDevice(SDL_Camera *device) return SDL_Unsupported(); } -static SDL_CameraFrameResult DUMMYCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult DUMMYCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { SDL_Unsupported(); return SDL_CAMERA_FRAME_ERROR; diff --git a/src/camera/emscripten/SDL_camera_emscripten.c b/src/camera/emscripten/SDL_camera_emscripten.c index 42c5aba4a1ee1..8eb67b7f845f6 100644 --- a/src/camera/emscripten/SDL_camera_emscripten.c +++ b/src/camera/emscripten/SDL_camera_emscripten.c @@ -39,7 +39,7 @@ static bool EMSCRIPTENCAMERA_WaitDevice(SDL_Camera *device) return false; } -static SDL_CameraFrameResult EMSCRIPTENCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult EMSCRIPTENCAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { void *rgba = SDL_malloc(device->actual_spec.width * device->actual_spec.height * 4); if (!rgba) { diff --git a/src/camera/mediafoundation/SDL_camera_mediafoundation.c b/src/camera/mediafoundation/SDL_camera_mediafoundation.c index a059b131feb15..4abe2294d4026 100644 --- a/src/camera/mediafoundation/SDL_camera_mediafoundation.c +++ b/src/camera/mediafoundation/SDL_camera_mediafoundation.c @@ -430,7 +430,7 @@ static void SDLCALL CleanupIMFMediaBuffer(void *userdata, void *value) SDL_free(objs); } -static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { SDL_assert(device->hidden->current_sample != NULL); @@ -562,7 +562,7 @@ static SDL_CameraFrameResult MEDIAFOUNDATION_CopyFrame(SDL_Surface *frame, const return SDL_CAMERA_FRAME_READY; } -static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult MEDIAFOUNDATION_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { SDL_assert(device->hidden->current_sample != NULL); diff --git a/src/camera/pipewire/SDL_camera_pipewire.c b/src/camera/pipewire/SDL_camera_pipewire.c index a5644736fe4fe..610cd71fabfab 100644 --- a/src/camera/pipewire/SDL_camera_pipewire.c +++ b/src/camera/pipewire/SDL_camera_pipewire.c @@ -577,7 +577,7 @@ static bool PIPEWIRECAMERA_WaitDevice(SDL_Camera *device) return true; } -static SDL_CameraFrameResult PIPEWIRECAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult PIPEWIRECAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { struct pw_buffer *b; diff --git a/src/camera/v4l2/SDL_camera_v4l2.c b/src/camera/v4l2/SDL_camera_v4l2.c index fe2491b61952e..0fd078bb309dc 100644 --- a/src/camera/v4l2/SDL_camera_v4l2.c +++ b/src/camera/v4l2/SDL_camera_v4l2.c @@ -125,7 +125,7 @@ static bool V4L2_WaitDevice(SDL_Camera *device) return false; } -static SDL_CameraFrameResult V4L2_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult V4L2_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { const int fd = device->hidden->fd; const io_method io = device->hidden->io; diff --git a/src/camera/vita/SDL_camera_vita.c b/src/camera/vita/SDL_camera_vita.c index 813f2a1a5fbfd..ff492b383afc6 100644 --- a/src/camera/vita/SDL_camera_vita.c +++ b/src/camera/vita/SDL_camera_vita.c @@ -190,7 +190,7 @@ static bool VITACAMERA_WaitDevice(SDL_Camera *device) return true; } -static SDL_CameraFrameResult VITACAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS) +static SDL_CameraFrameResult VITACAMERA_AcquireFrame(SDL_Camera *device, SDL_Surface *frame, Uint64 *timestampNS, int *rotation) { SceCameraRead read = {0}; read.size = sizeof(SceCameraRead); diff --git a/test/testcamera.c b/test/testcamera.c index b27092112a447..985852575160f 100644 --- a/test/testcamera.c +++ b/test/testcamera.c @@ -360,12 +360,17 @@ SDL_AppResult SDL_AppIterate(void *appstate) texture_updated = true; } + // the image might be coming from a mobile device that provides images in only one orientation, but the + // device might be rotated to a different one (like an iPhone providing portrait images even if you hold + // the phone in landscape mode). The rotation is how far to rotate the image clockwise to put it right-side + // up, for how the user would expect it to be for how they are holding the device. + const int rotation = (int) SDL_GetNumberProperty(SDL_GetSurfaceProperties(frame_current), SDL_PROP_SURFACE_ROTATION_NUMBER, 0); SDL_GetRenderOutputSize(renderer, &win_w, &win_h); d.x = ((win_w - texture->w) / 2.0f); d.y = ((win_h - texture->h) / 2.0f); d.w = (float)texture->w; d.h = (float)texture->h; - SDL_RenderTexture(renderer, texture, NULL, &d); + SDL_RenderTextureRotated(renderer, texture, NULL, &d, rotation, NULL, SDL_FLIP_NONE); } /* !!! FIXME: Render a "flip" icon if front_camera and back_camera are both != 0. */