Enable auth retry on clock skew error.

If a request gets an authentication error that indicates a clock skew
problem, clear the current authentication cached values and retry
the operation once.
Also, use an internal datetime that is one minute ahead, to help
align cached signature values across the valid time skew range.

Author: connelly38

driver/src/main/java/oracle/nosql/driver/AuthorizationProvider.java

@@ -71,4 +71,10 @@ public default void setRequiredHeaders(String authString,
             headers.set(AUTHORIZATION, authString);
         }
     }
+
+    /**
+     * Invalidate any cached authorization strings.
+     */
+    public default void flushCache() {
+    }
 }

driver/src/main/java/oracle/nosql/driver/http/Client.java

@@ -49,6 +49,7 @@
 
 import oracle.nosql.driver.AuthorizationProvider;
 import oracle.nosql.driver.DefaultRetryHandler;
+import oracle.nosql.driver.InvalidAuthorizationException;
 import oracle.nosql.driver.NoSQLException;
 import oracle.nosql.driver.NoSQLHandleConfig;
 import oracle.nosql.driver.RateLimiter;
@@ -678,6 +679,26 @@ public Result execute(Request kvRequest) {
                         rae);
                 throw new NoSQLException("Unexpected exception: " +
                         rae.getMessage(), rae);
+            } catch (InvalidAuthorizationException iae) {
+                /* allow a single retry on clock skew errors */
+                if (iae.getMessage().contains("clock skew") == false ||
+                    kvRequest.getNumRetries() > 0) {
+                    /* same as NoSQLException below */
+                    kvRequest.setRateLimitDelayedMs(rateDelayedMs);
+                    statsControl.observeError(kvRequest);
+                    logFine(logger, "Client execute NoSQLException: " +
+                            iae.getMessage());
+                    throw iae;
+                }
+                /* flush auth cache and do one retry */
+                authProvider.flushCache();
+                kvRequest.addRetryException(iae.getClass());
+                kvRequest.incrementRetries();
+                exception = iae;
+                logFine(logger,
+                        "Client retrying on InvalidAuthorizationException: " +
+                        iae.getMessage());
+                continue;
             } catch (SecurityInfoNotReadyException sinre) {
                 kvRequest.addRetryException(sinre.getClass());
                 exception = sinre;

driver/src/main/java/oracle/nosql/driver/iam/SignatureProvider.java

@@ -35,7 +35,6 @@
 import oracle.nosql.driver.SecurityInfoNotReadyException;
 import oracle.nosql.driver.iam.SecurityTokenSupplier.SecurityTokenBasedProvider;
 import oracle.nosql.driver.ops.Request;
-import oracle.nosql.driver.util.LruCache;
 
 import io.netty.handler.codec.http.HttpHeaders;
 
@@ -121,11 +120,6 @@ public class SignatureProvider
         "(request-target) host date opc-obo-token";
     private static final String OBO_TOKEN_HEADER = "opc-obo-token";
 
-    /* Cache key name */
-    private static final String CACHE_KEY = "signature";
-    /* Refresh key name */
-    private static final String REFRESH_CACHE_KEY = "refresh_signature";
-
     /* Maximum lifetime of signature 240 seconds */
     protected static final int MAX_ENTRY_LIFE_TIME = 240;
 
@@ -139,16 +133,21 @@ public class SignatureProvider
     /* Delegation token specified for signing */
     private String delegationToken;
 
-    private final LruCache<String, SignatureDetails> signatureCache;
+    /* the currently cached signature */
+    private SignatureDetails currentSigDetails;
+
+    /* new signature, in process of warmup */
+    private SignatureDetails refreshSigDetails;
 
     /* Refresh timer */
     private volatile Timer refresher;
 
-    /* Refresh time before signature expired from cache */
+    /* Refresh time before signature expires */
     private long refreshAheadMs = DEFAULT_REFRESH_AHEAD;
 
     /* Refresh interval, if zero, no refresh will be scheduled */
     private long refreshIntervalMs = 0;
+
     private String serviceHost;
     private Region region;
     private Logger logger;
@@ -675,8 +674,6 @@ protected SignatureProvider(AuthenticationProfileProvider profileProvider,
                 "Signature cannot be cached longer than " +
                 MAX_ENTRY_LIFE_TIME + " seconds");
         }
-        this.signatureCache =
-            new LruCache<String, SignatureDetails>(2, durationSeconds * 1000);
 
         this.refreshAheadMs = refreshAhead;
         long durationMS = durationSeconds * 1000;
@@ -739,6 +736,12 @@ public void setRequiredHeaders(String authString,
         }
     }
 
+    @Override
+    public synchronized void flushCache() {
+        currentSigDetails = null;
+        refreshSigDetails = null;
+    }
+
     /**
      * Get tenant OCID if using user principal.
      * @return tenant OCID of user
@@ -753,7 +756,6 @@ private String getTenantOCID() {
 
     @Override
     public void close() {
-        signatureCache.stop(false);
         if (refresher != null) {
             refresher.cancel();
             refresher = null;
@@ -858,28 +860,32 @@ private void logMessage(Level level, String msg) {
     }
 
     private SignatureDetails getSignatureDetails(Request request) {
-        String key = request.getIsRefresh() ? REFRESH_CACHE_KEY : CACHE_KEY;
-        SignatureDetails sigDetails = signatureCache.get(key);
+        SignatureDetails sigDetails =
+            (request.getIsRefresh() ? refreshSigDetails : currentSigDetails);
         if (sigDetails != null) {
             return sigDetails;
         }
 
         if (request.getIsRefresh()) {
-            /* try normal key before failing */
-            sigDetails = signatureCache.get(CACHE_KEY);
+            /* try current details before failing */
+            sigDetails = currentSigDetails;
             if (sigDetails != null) {
                 return sigDetails;
             }
         }
 
-        logMessage(Level.WARNING, "No signature in cache");
         return getSignatureDetailsInternal(false);
     }
 
     /* visible for testing */
     synchronized SignatureDetails getSignatureDetailsInternal(boolean isRefresh)
     {
-        String date = createFormatter().format(new Date());
+        /*
+         * add one minute to the current time, so that any caching is
+         * effective over a more valid time period.
+         */
+        long nowPlus = System.currentTimeMillis() + 60_000L;
+        String date = createFormatter().format(new Date(nowPlus));
         String keyId = provider.getKeyId();
         if (provider instanceof InstancePrincipalsProvider) {
             privateKeyProvider.reload(provider.getPrivateKey(),
@@ -907,7 +913,7 @@ synchronized SignatureDetails getSignatureDetailsInternal(boolean isRefresh)
              * if this is not a refresh, use the normal key and schedule a
              * refresh
              */
-            signatureCache.put(CACHE_KEY, sigDetails);
+            currentSigDetails = sigDetails;
             scheduleRefresh();
         } else {
             /*
@@ -916,16 +922,15 @@ synchronized SignatureDetails getSignatureDetailsInternal(boolean isRefresh)
              * 1. perform callbacks if needed and when done,
              * 2. move the object to the normal key and schedule a refresh
              */
-            signatureCache.put(REFRESH_CACHE_KEY, sigDetails);
+            refreshSigDetails = sigDetails;
         }
         return sigDetails;
     }
 
     private synchronized void setRefreshKey() {
-        SignatureDetails sigDetails =
-            signatureCache.remove(REFRESH_CACHE_KEY);
-        if (sigDetails != null) {
-            signatureCache.put(CACHE_KEY, sigDetails);
+        if (refreshSigDetails != null) {
+            currentSigDetails = refreshSigDetails;
+            refreshSigDetails = null;
         }
     }
 
@@ -978,10 +983,9 @@ private class RefreshTask extends TimerTask {
         private static final int DELAY_MS = 500;
 
         private void handleRefreshCallback(long refreshMs) {
-            SignatureDetails sigDetails = signatureCache.get(REFRESH_CACHE_KEY);
-            if (sigDetails == null) {
+            if (refreshSigDetails == null) {
                 logMessage(Level.FINE,
-                           "Refresh didn't find refresh cache key");
+                           "Refresh didn't find cached refresh key");
                 return;
             }
 

driver/src/main/java/oracle/nosql/driver/ops/Request.java

@@ -84,7 +84,7 @@ public int getTimeoutInternal() {
     /**
      * @hidden
      * this is public to allow access from Client during refresh
-     * @param timeoutMs timeout in milliseconds
+     * @param timeoutMs the request timeout, in milliseconds
      */
     public void setTimeoutInternal(int timeoutMs) {
         if (timeoutMs <= 0) {
@@ -455,7 +455,7 @@ public boolean getIsRefresh() {
     /**
      * @hidden
      * Copy internal fields to another Request object.
-     * @param other the request to copy
+     * @param other the Request object to copy to.
      */
     public void copyTo(Request other) {
         other.setTimeoutInternal(this.timeoutMs);