diff --git a/include/SDL3/SDL_mouse.h b/include/SDL3/SDL_mouse.h
index 79ce61a507..b190e9d435 100644
--- a/include/SDL3/SDL_mouse.h
+++ b/include/SDL3/SDL_mouse.h
@@ -415,13 +415,19 @@ extern SDL_DECLSPEC bool SDLCALL SDL_WarpMouseGlobal(float x, float y);
 /**
  * Set a user-defined function by which to transform relative mouse inputs.
  * This overrides the relative system scale and relative speed scale hints.
+ * Should be called prior to enabling relative mouse mode, fails otherwise.
+ * 
+ * \param callback a callback used to transform relative mouse motion, 
+                   or NULL for default behavior.
+ * \param userdata a pointer that will be passed to `callback`.
+ * \returns true on success or false on failure; call SDL_GetError() for more
+ *          information.
  *
- * \param callback a callback used to transform relative mouse motion, or NULL for default behavior.
- * \param userdata a pointer that is passed to `callback`.
+ * \threadsafety This function should only be called on the main thread.
  *
  * \since This function is available since SDL 3.2.6.
  */
-extern SDL_DECLSPEC void SDLCALL SDL_SetRelativeMouseTransform(SDL_MouseMotionTransformCallback callback, void *userdata);
+extern SDL_DECLSPEC bool SDLCALL SDL_SetRelativeMouseTransform(SDL_MouseMotionTransformCallback callback, void *userdata);
 
 /**
  * Set relative mouse mode for a window.
diff --git a/src/events/SDL_mouse.c b/src/events/SDL_mouse.c
index 7fcb097f34..e2ce07ffd6 100644
--- a/src/events/SDL_mouse.c
+++ b/src/events/SDL_mouse.c
@@ -1120,11 +1120,15 @@ void SDL_QuitMouse(void)
     SDL_mice = NULL;
 }
 
-void SDL_SetRelativeMouseTransform(SDL_MouseMotionTransformCallback transform, void *userdata)
+bool SDL_SetRelativeMouseTransform(SDL_MouseMotionTransformCallback transform, void *userdata)
 {
     SDL_Mouse *mouse = SDL_GetMouse();
+    if (mouse->relative_mode) {
+        return SDL_SetError("Can't set mouse transform while relative mode is active");
+    }
     mouse->InputTransform = transform;
     mouse->input_transform_data = userdata;
+    return true;
 }
 
 SDL_MouseButtonFlags SDL_GetMouseState(float *x, float *y)