fix(oauth): 🐛 prevent deadlock and token desync for rotating refresh tokens

This commit fixes critical issues in OAuth providers (Google, iFlow, Qwen) that use rotating refresh tokens, where each token refresh invalidates the previous token.

**Problems Fixed:**

1. **Deadlock Prevention**: Removed inline re-authentication from `refresh_token()` method that was called while holding a lock. When refresh failed with HTTP 400/401/403, the method would call `initialize_token()` directly, which would try to acquire the same lock, causing a deadlock. Now, invalid token errors are caught and queued for background re-authentication via `asyncio.create_task()`.

2. **Token Desync**: Changed `_save_credentials()` to write to disk FIRST, then update cache. Previously, cache was updated first with `buffer_on_failure=True`, which could leave stale tokens on disk if the write failed. For rotating tokens, this caused the old refresh_token on disk to become invalid after a successful API call, requiring re-auth on restart.

3. **Stale Cache Usage**: Modified `refresh_token()` to always read fresh credentials from disk before refreshing, preventing use of stale cached tokens that may have been invalidated by another process.

4. **New Error Type**: Introduced `CredentialNeedsReauthError` exception to signal rotatable authentication failures. This allows the client to rotate to the next credential without logging scary tracebacks, while background re-auth fixes the broken credential.

**Changes:**

- Add `CredentialNeedsReauthError` exception class and classification in error_handler.py
- Catch and wrap `CredentialNeedsReauthError` in client.py retry loop
- Replace inline re-auth with background task queuing in all OAuth providers
- Change `_save_credentials()` to disk-first writes with no buffering for rotating tokens
- Add `force_interactive` parameter to `initialize_token()` for explicit re-auth requests
- Always reload credentials from disk before refresh to prevent stale token usage
- Return boolean from `_save_credentials()` and raise IOError on critical failures
- Update re-auth queue processing to call `initialize_token(force_interactive=True)`

Author: Mirrowel

src/rotator_library/client.py

@@ -23,6 +23,7 @@
 from .failure_logger import log_failure, configure_failure_logger
 from .error_handler import (
     PreRequestCallbackError,
+    CredentialNeedsReauthError,
     classify_error,
     AllProviders,
     NoAvailableKeysError,
@@ -755,6 +756,12 @@ async def _safe_streaming_wrapper(
                         await self.usage_manager.record_success(key, model)
                     break
 
+                except CredentialNeedsReauthError as e:
+                    # This credential needs re-authentication but re-auth is already queued.
+                    # Wrap it so the outer retry loop can rotate to the next credential.
+                    # No scary traceback needed - this is an expected recovery scenario.
+                    raise StreamedAPIError("Credential needs re-authentication", data=e)
+
                 except (
                     litellm.RateLimitError,
                     litellm.ServiceUnavailableError,

src/rotator_library/error_handler.py

@@ -117,6 +117,31 @@ class PreRequestCallbackError(Exception):
     pass
 
 
+class CredentialNeedsReauthError(Exception):
+    """
+    Raised when a credential's refresh token is invalid and re-authentication is required.
+
+    This is a rotatable error - the request should try the next credential while
+    the broken credential is queued for re-authentication in the background.
+
+    Unlike generic HTTPStatusError, this exception signals:
+    - The credential is temporarily unavailable (needs user action)
+    - Re-auth has already been queued
+    - The request should rotate to the next credential without logging scary tracebacks
+
+    Attributes:
+        credential_path: Path to the credential file that needs re-auth
+        message: Human-readable message about the error
+    """
+
+    def __init__(self, credential_path: str, message: str = ""):
+        self.credential_path = credential_path
+        self.message = (
+            message or f"Credential '{credential_path}' requires re-authentication"
+        )
+        super().__init__(self.message)
+
+
 # =============================================================================
 # ERROR TRACKING FOR CLIENT REPORTING
 # =============================================================================
@@ -698,6 +723,14 @@ def classify_error(e: Exception, provider: Optional[str] = None) -> ClassifiedEr
             status_code=400,  # Treat as a bad request
         )
 
+    if isinstance(e, CredentialNeedsReauthError):
+        # This is a rotatable error - credential is broken but re-auth is queued
+        return ClassifiedError(
+            error_type="credential_reauth_needed",
+            original_exception=e,
+            status_code=401,  # Treat as auth error for reporting purposes
+        )
+
     if isinstance(e, RateLimitError):
         retry_after = get_retry_after(e)
         # Check if this is a quota error vs rate limit
@@ -789,6 +822,7 @@ def should_rotate_on_error(classified_error: ClassifiedError) -> bool:
     - quota_exceeded: Current key/account exhausted
     - forbidden: Current credential denied access
     - authentication: Current credential invalid
+    - credential_reauth_needed: Credential needs interactive re-auth (queued)
     - server_error: Provider having issues (might work with different endpoint/key)
     - api_connection: Network issues (might be transient)
     - unknown: Safer to try another key

src/rotator_library/providers/google_oauth_base.py

@@ -22,6 +22,7 @@
 from ..utils.headless_detection import is_headless_environment
 from ..utils.reauth_coordinator import get_reauth_coordinator
 from ..utils.resilient_io import safe_write_json
+from ..error_handler import CredentialNeedsReauthError
 
 lib_logger = logging.getLogger("rotator_library")
 
@@ -366,7 +367,6 @@ async def _refresh_token(
             max_retries = 3
             new_token_data = None
             last_error = None
-            needs_reauth = False
 
             async with httpx.AsyncClient() as client:
                 for attempt in range(max_retries):
@@ -390,15 +390,42 @@ async def _refresh_token(
                     except httpx.HTTPStatusError as e:
                         last_error = e
                         status_code = e.response.status_code
-
-                        # [INVALID GRANT HANDLING] Handle 401/403 by triggering re-authentication
-                        if status_code == 401 or status_code == 403:
-                            lib_logger.warning(
-                                f"Refresh token invalid for '{Path(path).name}' (HTTP {status_code}). "
-                                f"Token may have been revoked or expired. Starting re-authentication..."
+                        error_body = e.response.text
+
+                        # [INVALID GRANT HANDLING] Handle 400/401/403 by queuing for re-auth
+                        # We must NOT call initialize_token from here as we hold a lock (would deadlock)
+                        if status_code == 400:
+                            # Check if this is an invalid_grant error
+                            if "invalid_grant" in error_body.lower():
+                                lib_logger.info(
+                                    f"Credential '{Path(path).name}' needs re-auth (HTTP 400: invalid_grant). "
+                                    f"Queued for re-authentication, rotating to next credential."
+                                )
+                                asyncio.create_task(
+                                    self._queue_refresh(
+                                        path, force=True, needs_reauth=True
+                                    )
+                                )
+                                raise CredentialNeedsReauthError(
+                                    credential_path=path,
+                                    message=f"Refresh token invalid for '{Path(path).name}'. Re-auth queued.",
+                                )
+                            else:
+                                # Other 400 error - raise it
+                                raise
+
+                        elif status_code in (401, 403):
+                            lib_logger.info(
+                                f"Credential '{Path(path).name}' needs re-auth (HTTP {status_code}). "
+                                f"Queued for re-authentication, rotating to next credential."
+                            )
+                            asyncio.create_task(
+                                self._queue_refresh(path, force=True, needs_reauth=True)
+                            )
+                            raise CredentialNeedsReauthError(
+                                credential_path=path,
+                                message=f"Token invalid for '{Path(path).name}' (HTTP {status_code}). Re-auth queued.",
                             )
-                            needs_reauth = True
-                            break  # Exit retry loop to trigger re-auth
 
                         elif status_code == 429:
                             # Rate limit - honor Retry-After header if present
@@ -438,23 +465,6 @@ async def _refresh_token(
                             continue
                         raise
 
-            # [INVALID GRANT RE-AUTH] Trigger OAuth flow if refresh token is invalid
-            if needs_reauth:
-                lib_logger.info(
-                    f"Starting re-authentication for '{Path(path).name}'..."
-                )
-                try:
-                    # Call initialize_token to trigger OAuth flow
-                    new_creds = await self.initialize_token(path)
-                    return new_creds
-                except Exception as reauth_error:
-                    lib_logger.error(
-                        f"Re-authentication failed for '{Path(path).name}': {reauth_error}"
-                    )
-                    raise ValueError(
-                        f"Refresh token invalid and re-authentication failed: {reauth_error}"
-                    )
-
             # If we exhausted retries without success
             if new_token_data is None:
                 raise last_error or Exception("Token refresh failed after all retries")
@@ -832,7 +842,7 @@ async def _process_reauth_queue(self):
 
                 try:
                     lib_logger.info(f"Starting re-auth for '{Path(path).name}'...")
-                    await self.initialize_token(path)
+                    await self.initialize_token(path, force_interactive=True)
                     lib_logger.info(f"Re-auth SUCCESS for '{Path(path).name}'")
 
                 except Exception as e:
@@ -1058,14 +1068,22 @@ async def handle_callback(reader, writer):
         return new_creds
 
     async def initialize_token(
-        self, creds_or_path: Union[Dict[str, Any], str]
+        self,
+        creds_or_path: Union[Dict[str, Any], str],
+        force_interactive: bool = False,
     ) -> Dict[str, Any]:
         """
         Initialize OAuth token, triggering interactive OAuth flow if needed.
 
         If interactive OAuth is required (expired refresh token, missing credentials, etc.),
         the flow is coordinated globally via ReauthCoordinator to ensure only one
         interactive OAuth flow runs at a time across all providers.
+
+        Args:
+            creds_or_path: Either a credentials dict or path to credentials file.
+            force_interactive: If True, skip expiry checks and force interactive OAuth.
+                               Use this when the refresh token is known to be invalid
+                               (e.g., after HTTP 400 from token endpoint).
         """
         path = creds_or_path if isinstance(creds_or_path, str) else None
 
@@ -1085,7 +1103,11 @@ async def initialize_token(
                 await self._load_credentials(creds_or_path) if path else creds_or_path
             )
             reason = ""
-            if not creds.get("refresh_token"):
+            if force_interactive:
+                reason = (
+                    "re-authentication was explicitly requested (refresh token invalid)"
+                )
+            elif not creds.get("refresh_token"):
                 reason = "refresh token is missing"
             elif self._is_token_expired(creds):
                 reason = "token is expired"