- Added new method in NoSQLHandle: queryIterable() to return the results of a query in an iterable/iterator format. The returned QueryIterableResult should be used in a try-with-resources statement to ensure proper closing of resources.

- Updated NoSQLHandle interface to extend AutoClosable.

- Updated examples to show new usage of the queryIterable() method and auto-closable handler.

- Fixed a bug in test infrastructure InternalsTest when other tables are present.

Author: cezarfx

CHANGELOG.md

@@ -2,6 +2,15 @@
 All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/).
 
+## Unreleased
+
+### Added
+- added new method in NoSQLHandle queryIterable to return the results of a
+query in an iterable/iterator format. The returned QueryIterableResult should
+be used in a try-with-resources statement to ensure proper closing of
+resources.
+- updated NoSQLHandle interface to extend AutoClosable
+
 ## [5.3.2] 2022-03-21
 
 ### Added

README.md

@@ -184,6 +184,7 @@ is required if using Instance Principal or Resource Principal authorization.
  */
 
 import java.io.IOException;
+import java.util.ArrayList;
 
 import oracle.nosql.driver.AuthorizationProvider;
 import oracle.nosql.driver.NoSQLHandle;
@@ -193,8 +194,11 @@ import oracle.nosql.driver.iam.SignatureProvider;
 import oracle.nosql.driver.kv.StoreAccessTokenProvider;
 import oracle.nosql.driver.ops.GetRequest;
 import oracle.nosql.driver.ops.GetResult;
+import oracle.nosql.driver.ops.QueryRequest;
+import oracle.nosql.driver.ops.QueryResult;
 import oracle.nosql.driver.ops.PutRequest;
 import oracle.nosql.driver.ops.PutResult;
+import oracle.nosql.driver.ops.QueryIterableResult;
 import oracle.nosql.driver.ops.Request;
 import oracle.nosql.driver.ops.TableLimits;
 import oracle.nosql.driver.ops.TableRequest;
@@ -206,6 +210,8 @@ import oracle.nosql.driver.values.MapValue;
  * - create a table
  * - put a row
  * - get a row
+ * - run a query using iterable/iterator
+ * - run a query using partial results
  * - drop the table
  *
  * See the examples for more interesting operations. This quickstart is
@@ -378,8 +384,7 @@ public class Quickstart {
          * Configure and get a NoSQLHandle. All service specific configuration
          * is handled here
          */
-        NoSQLHandle handle = qs.getHandle();
-        try {
+        try (NoSQLHandle handle = qs.getHandle()) {
 
             /*
              * Create a simple table with an integer key, string name and
@@ -427,11 +432,29 @@ public class Quickstart {
             System.out.println("Got row, result " + getRes);
 
             /*
-             * Perform a query
+             * Perform a query using iterable and iterator
              */
             QueryRequest queryRequest = new QueryRequest()
                 .setStatement("select * from " + tableName);
 
+            /*
+             * To ensure the query resources are closed properly, use
+             * try-with-resources statement.
+             */
+            try (QueryIterableResult results = 
+                    handle.queryIterable(queryRequest)) {
+                System.out.println("Query results:");
+                for (MapValue res : results) {
+                    System.out.println("\t" + res);
+                }
+            }
+            
+            /*
+             * Perform a query using partial results
+             */
+            queryRequest = new QueryRequest()
+                .setStatement("select * from " + tableName);
+
             /*
              * Because a query can return partial results execution must occur
              * in a loop, accumulating or processing results
@@ -441,7 +464,7 @@ public class Quickstart {
                 QueryResult queryResult = handle.query(queryRequest);
                 results.addAll(queryResult.getResults());
             } while (!queryRequest.isDone());
-            System.out.println("Query results:");
+            System.out.println("Query results again:");
             for (MapValue res : results) {
                 System.out.println("\t" + res);
             }
@@ -456,9 +479,6 @@ public class Quickstart {
                                   20000,
                                   1000);
             System.out.println("Dropped table " + tableName + ", done...");
-        } finally {
-            /* Shutdown handle so the process can exit. */
-            handle.close();
         }
     }
 }
@@ -468,7 +488,8 @@ public class Quickstart {
 
 Several example programs are provided in the examples directory to
 illustrate the API. They can be found in the release download from GitHub or
-directly in [GitHub NoSQL Examples](https://github.com/oracle/nosql-java-sdk/tree/master/examples). These examples can be run against the Oracle NoSQL
+directly in [GitHub NoSQL Examples](https://github.com/oracle/nosql-java-sdk/tree/main/examples). These examples can be run 
+against the Oracle NoSQL
 Database, the NoSQL Database Cloud Service or an instance of the Oracle
 NoSQL Cloud Simulator. The code that differentiates among the configurations
 is in the file Common.java and can be examined to understand the differences.

driver/src/main/java/oracle/nosql/driver/NoSQLHandle.java

@@ -7,9 +7,6 @@
 
 package oracle.nosql.driver;
 
-import oracle.nosql.driver.ops.SystemRequest;
-import oracle.nosql.driver.ops.SystemResult;
-import oracle.nosql.driver.ops.SystemStatusRequest;
 import oracle.nosql.driver.ops.DeleteRequest;
 import oracle.nosql.driver.ops.DeleteResult;
 import oracle.nosql.driver.ops.GetIndexesRequest;
@@ -26,10 +23,14 @@
 import oracle.nosql.driver.ops.PutRequest;
 import oracle.nosql.driver.ops.PutRequest.Option;
 import oracle.nosql.driver.ops.PutResult;
+import oracle.nosql.driver.ops.QueryIterableResult;
 import oracle.nosql.driver.ops.QueryRequest;
 import oracle.nosql.driver.ops.QueryResult;
 import oracle.nosql.driver.ops.Request;
 import oracle.nosql.driver.ops.Result;
+import oracle.nosql.driver.ops.SystemRequest;
+import oracle.nosql.driver.ops.SystemResult;
+import oracle.nosql.driver.ops.SystemStatusRequest;
 import oracle.nosql.driver.ops.TableRequest;
 import oracle.nosql.driver.ops.TableResult;
 import oracle.nosql.driver.ops.TableUsageRequest;
@@ -116,7 +117,7 @@
  * threads.
  * </p>
  */
-public interface NoSQLHandle {
+public interface NoSQLHandle extends AutoCloseable {
 
     /**
      * Deletes a row from a table. The row is identified using a primary key
@@ -311,6 +312,54 @@ public interface NoSQLHandle {
      */
     QueryResult query(QueryRequest request);
 
+    /**
+     * Queries a table based on the query statement specified in the
+     * {@link QueryRequest} while returning an iterable result.
+     *
+     * Queries that include a full shard key will execute much more efficiently
+     * than more distributed queries that must go to multiple shards.
+     * <p>
+     * Remote calls, including preparation of a query statement, will not
+     * occur until the iteration starts. This means that errors such as an
+     * invalid statement or network issue will be thrown from the iterator
+     * and not this method.
+     * <p>
+     * Table- and system-style queries such as "CREATE TABLE ..." or "DROP TABLE .."
+     * are not supported by this interfaces. Those operations must be performed using
+     * {@link #tableRequest} or {@link #systemRequest} as appropriate.
+     * <p>
+     * The results are returned through an iterator, if connected to the
+     * cloud, the SDK uses the read/write rate limits set in the
+     * NoSQLHandleConfig or they can be overwritten using the
+     * {@link Request#setReadRateLimiter(RateLimiter)} and
+     * {@link Request#setWriteRateLimiter(RateLimiter)}.
+     * <p>
+     * Note: Since iterators might use resources until they reach the end, it
+     * is necessary to close the QueryIterableResult or use the
+     * try-with-resources statement:
+     * <p><pre>
+     *    QueryRequest qreq = new QueryRequest()
+     *        .setStatement("select * from MyTable");
+     *
+     *    try (QueryIterableResult qir = handle.queryIterable(qreq)) {
+     *        for( MapValue row : qir) {
+     *            // do something with row
+     *        }
+     *    }
+     * </pre>
+     *
+     * @param request the input parameters for the operation
+     *
+     * @return the iterable result of the operation
+     *
+     * @throws IllegalArgumentException if any of the parameters are invalid or
+     * required parameters are missing
+     *
+     * @throws NoSQLException if the operation cannot be performed for any other
+     * reason
+     */
+    QueryIterableResult queryIterable(QueryRequest request);
+
     /**
      * Prepares a query for execution and reuse. See {@link #query} for general
      * information and restrictions. It is recommended that prepared queries
@@ -374,9 +423,9 @@ public interface NoSQLHandle {
      * @throws NoSQLException if the operation cannot be performed for
      * any other reason
      */
-    public TableResult doTableRequest(TableRequest request,
-                                      int timeoutMs,
-                                      int pollIntervalMs);
+    TableResult doTableRequest(TableRequest request,
+                               int timeoutMs,
+                               int pollIntervalMs);
 
     /**
      * On-premise only.
@@ -567,9 +616,9 @@ public TableResult doTableRequest(TableRequest request,
      * @throws NoSQLException if the operation cannot be performed for
      * any other reason
      */
-    public SystemResult doSystemRequest(String statement,
-                                        int timeoutMs,
-                                        int pollIntervalMs);
+    SystemResult doSystemRequest(String statement,
+                                 int timeoutMs,
+                                 int pollIntervalMs);
 
     /**
      * Returns an object that allows control over how statistics are collected.
@@ -578,7 +627,7 @@ public SystemResult doSystemRequest(String statement,
      *
      * @since 5.2.30
      */
-    public StatsControl getStatsControl();
+    StatsControl getStatsControl();
 
     /**
      * Closes the handle, releasing its memory and network resources. Once

driver/src/main/java/oracle/nosql/driver/http/NoSQLHandleImpl.java

@@ -9,6 +9,7 @@
 
 import java.util.ArrayList;
 import java.util.logging.Logger;
+
 import javax.net.ssl.SSLException;
 
 import oracle.nosql.driver.AuthorizationProvider;
@@ -33,6 +34,7 @@
 import oracle.nosql.driver.ops.PrepareResult;
 import oracle.nosql.driver.ops.PutRequest;
 import oracle.nosql.driver.ops.PutResult;
+import oracle.nosql.driver.ops.QueryIterableResult;
 import oracle.nosql.driver.ops.QueryRequest;
 import oracle.nosql.driver.ops.QueryResult;
 import oracle.nosql.driver.ops.SystemRequest;
@@ -198,6 +200,12 @@ public QueryResult query(QueryRequest request) {
         return (QueryResult) client.execute(request);
     }
 
+    @Override
+    public QueryIterableResult queryIterable(QueryRequest request) {
+        checkClient();
+        return new QueryIterableResult(request, this);
+    }
+
     @Override
     public PrepareResult prepare(PrepareRequest request) {
         checkClient();

driver/src/main/java/oracle/nosql/driver/ops/DurableRequest.java

@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2011, 2021 Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2011, 2022 Oracle and/or its affiliates. All rights reserved.
  *
  * Licensed under the Universal Permissive License v 1.0 as shown at
  *  https://oss.oracle.com/licenses/upl/

driver/src/main/java/oracle/nosql/driver/ops/QueryRequest.java

@@ -28,6 +28,25 @@
  * may be reused. This is because prepared queries bypass query compilation.
  * They also allow for parameterized queries using bind variables.
  * <p>
+ * There are two ways to get the results of a query: using an iterator or
+ * loop through partial results.
+ * <p>
+ * <b>Iterator</b>
+ * <p>
+ * Use {@link NoSQLHandle#queryIterable(QueryRequest)} to get an iterable
+ * that contains all the results. Usage example:
+ * <pre>
+ *    NoSQLHandle handle = ...;
+ *
+ *    QueryRequest qreq = new QueryRequest().setStatement("select * from foo");
+ *
+ *    for (MapValue row : handle.queryIterable(qreq) ) {
+ *        // do something with row
+ *    }
+ * </pre>
+ * <p>
+ * <b>Partial results</b>
+ * <p>
  * To compute and retrieve the full result set of a query, the same QueryRequest
  * instance will, in general, have to be executed multiple times (via
  * {@link NoSQLHandle#query}. Each execution returns a {@link QueryResult},
@@ -60,8 +79,9 @@
  * application threads need to run the same query concurrently, they must
  * create and use their own QueryRequest instances.
  *
- * @see NoSQLHandle#query
- * @see NoSQLHandle#prepare
+ * @see NoSQLHandle#queryIterable(QueryRequest)
+ * @see NoSQLHandle#query(QueryRequest)
+ * @see NoSQLHandle#prepare(PrepareRequest)
  */
 public class QueryRequest extends Request {
 
@@ -129,6 +149,20 @@ public QueryRequest copyInternal() {
         return internalReq;
     }
 
+    /**
+     * @hidden
+     * Creates a copy that starts fresh from the beginning.
+     * @return a copy of the instance in a new object
+     */
+    public QueryRequest copy() {
+        QueryRequest internalReq = copyInternal();
+        internalReq.statement = statement;
+        internalReq.isInternal = false;
+        internalReq.shardId = -1;
+        internalReq.driver = null;
+        return internalReq;
+    }
+
     /**
      * @hidden
      *
@@ -295,7 +329,7 @@ public QueryRequest setCompartment(String compartment) {
 
     /**
      * Returns the limit on number of items returned by the operation. If
-     * not set by the application this value will be 0 which means no limit.
+     * not set by the application this value will be 0 which means no limit set.
      *
      * @return the limit, or 0 if not set
      */

driver/src/main/java/oracle/nosql/driver/ops/RetryStats.java

@@ -51,7 +51,20 @@ public RetryStats() {
      * @param e the exception class
      */
     public void addException(Class<? extends Throwable> e) {
-        int i = getNumExceptions(e) + 1;
+        addException(e, 1);
+    }
+
+    /**
+     * @hidden
+     * Internal use only.
+     * Adds an exception class to the stats object.
+     * This increments the exception count and adds to the count of
+     * this type of exception class.
+     * @param e the exception class
+     * @param n the number of such exceptions
+     */
+    public void addException(Class<? extends Throwable> e, int n) {
+        int i = getNumExceptions(e) + n;
         exceptionMap.put(e, i);
     }
 

driver/src/main/java/oracle/nosql/driver/query/QueryDriver.java

@@ -15,8 +15,6 @@
 import oracle.nosql.driver.ops.PreparedStatement;
 import oracle.nosql.driver.ops.QueryRequest;
 import oracle.nosql.driver.ops.QueryResult;
-import oracle.nosql.driver.query.PlanIter;
-import oracle.nosql.driver.query.RuntimeControlBlock;
 import oracle.nosql.driver.values.FieldValue;
 import oracle.nosql.driver.values.MapValue;
 

driver/src/test/java/oracle/nosql/driver/InternalsTest.java

@@ -12,11 +12,11 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import oracle.nosql.driver.ops.ListTablesRequest;
+
 import org.junit.Before;
 import org.junit.Test;
 
-import oracle.nosql.driver.ops.ListTablesRequest;
-
 public class InternalsTest extends ProxyTestBase {
 
     @Override
@@ -32,6 +32,7 @@ protected void perTestHandleConfig(NoSQLHandleConfig config) {
      * is run. The default runs a ListTables request.
      */
     public void beforeTest() throws Exception {
+        super.beforeTest();
         handle = getHandle(endpoint);
         /* do NOT list tables here */
     }