Added support for the ArrowArrayStream PyCapsule interface.

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -35,6 +35,8 @@ Common Changes
 
 #)  Changes to :ref:`data frame <dataframeformat>` support:
 
+    - Added internal support for the ArrowArrayStream PyCapsule interface to
+      simplify :ref:`OracleDataFrame <oracledataframeobj>` use.
     - Remove use of the DataFrame Interchange Protocol in
       :ref:`OracleDataFrames <oracledataframeobj>`.
     - Documentation on methods and attributes on the ``DataFrame`` and

samples/dataframe_pandas.py

@@ -60,9 +60,7 @@
 odf = connection.fetch_df_all(statement=sql, arraysize=100)
 
 # Get a Pandas DataFrame from the data
-df = pyarrow.Table.from_arrays(
-    odf.column_arrays(), names=odf.column_names()
-).to_pandas()
+df = pyarrow.table(odf).to_pandas()
 
 # Perform various Pandas operations on the DataFrame
 
@@ -93,9 +91,7 @@
 # behavior on the sample table.
 sql = "select id, name from SampleQueryTab order by id"
 for odf in connection.fetch_df_batches(statement=sql, size=10):
-    df_b = pyarrow.Table.from_arrays(
-        odf.column_arrays(), names=odf.column_names()
-    ).to_pandas()
+    df_b = pyarrow.table(odf).to_pandas()
     print(f"Appending {df_b.shape[0]} rows")
     df = pandas.concat([df, df_b], ignore_index=True)
 
@@ -137,9 +133,7 @@
 odf = connection.fetch_df_all(statement=sql, arraysize=100)
 
 # Get a Pandas DataFrame from the data
-df = pyarrow.Table.from_arrays(
-    odf.column_arrays(), names=odf.column_names()
-).to_pandas()
+df = pyarrow.table(odf).to_pandas()
 
 # Perform various Pandas operations on the DataFrame
 

samples/dataframe_pandas_async.py

@@ -64,9 +64,7 @@ async def main():
     odf = await connection.fetch_df_all(statement=SQL, arraysize=100)
 
     # Get a Pandas DataFrame from the data
-    df = pyarrow.Table.from_arrays(
-        odf.column_arrays(), names=odf.column_names()
-    ).to_pandas()
+    df = pyarrow.table(odf).to_pandas()
 
     # Perform various Pandas operations on the DataFrame
 
@@ -96,9 +94,7 @@ async def main():
     # Tune 'size' for your data set. Here it is small to show the batch fetch
     # behavior on the sample table.
     async for odf in connection.fetch_df_batches(statement=SQL, size=10):
-        df_b = pyarrow.Table.from_arrays(
-            odf.column_arrays(), names=odf.column_names()
-        ).to_pandas()
+        df_b = pyarrow.table(odf).to_pandas()
         print(f"Appending {df_b.shape[0]} rows")
         df = pandas.concat([df, df_b], ignore_index=True)
 
@@ -141,9 +137,7 @@ async def main():
     odf = await connection.fetch_df_all(statement=sql, arraysize=100)
 
     # Get a Pandas DataFrame from the data
-    df = pyarrow.Table.from_arrays(
-        odf.column_arrays(), names=odf.column_names()
-    ).to_pandas()
+    df = pyarrow.table(odf).to_pandas()
 
     # Perform various Pandas operations on the DataFrame
 

samples/dataframe_parquet_write.py

@@ -61,9 +61,7 @@
 
 for odf in connection.fetch_df_batches(statement=SQL, size=FETCH_BATCH_SIZE):
 
-    pyarrow_table = pyarrow.Table.from_arrays(
-        arrays=odf.column_arrays(), names=odf.column_names()
-    )
+    pyarrow_table = pyarrow.table(odf)
 
     if not pqwriter:
         pqwriter = pq.ParquetWriter(PARQUET_FILE_NAME, pyarrow_table.schema)

samples/dataframe_polars.py

@@ -57,10 +57,7 @@
 odf = connection.fetch_df_all(statement=SQL1, arraysize=100)
 
 # Convert to a Polars DataFrame
-pyarrow_table = pyarrow.Table.from_arrays(
-    odf.column_arrays(), names=odf.column_names()
-)
-p = polars.from_arrow(pyarrow_table)
+p = polars.from_arrow(odf)
 
 print(type(p))  # <class 'polars.dataframe.frame.DataFrame'>
 

samples/dataframe_pyarrow.py

@@ -56,9 +56,7 @@
 odf = connection.fetch_df_all(statement=SQL1, arraysize=100)
 
 # Create a PyArrow table
-pyarrow_table = pyarrow.Table.from_arrays(
-    arrays=odf.column_arrays(), names=odf.column_names()
-)
+pyarrow_table = pyarrow.table(odf)
 
 print("Type:")
 print(type(pyarrow_table))  # <class 'pyarrow.lib.Table'>

src/oracledb/__init__.py

@@ -287,6 +287,7 @@
 
 from .utils import (
     enable_thin_mode as enable_thin_mode,
+    from_arrow as from_arrow,
     register_params_hook as register_params_hook,
     register_password_type as register_password_type,
     register_protocol as register_protocol,

src/oracledb/arrow_array.py

@@ -29,6 +29,8 @@
 # array data to other data frame libraries.
 # -----------------------------------------------------------------------------
 
+from .arrow_impl import ArrowArrayImpl
+
 from . import errors
 
 
@@ -51,6 +53,12 @@ def __repr__(self):
     def __str__(self):
         return self.__repr__()
 
+    @classmethod
+    def _from_arrow(cls, obj):
+        array = cls.__new__(cls)
+        array._impl = ArrowArrayImpl.from_arrow_array(obj)
+        return array
+
     @classmethod
     def _from_impl(cls, impl):
         array = cls.__new__(cls)

src/oracledb/arrow_impl.pxd

@@ -46,10 +46,18 @@ cdef extern from "nanoarrow.h":
         ArrowArray** children
         const void** buffers
         void (*release)(ArrowArray*)
+        void *private_data
 
     cdef struct ArrowSchema:
+        const char *format;
+        const char *name;
+        const char *metadata;
+        int64_t flags
+        int64_t n_children
         ArrowSchema** children
+        ArrowSchema* dictionary
         void (*release)(ArrowSchema*)
+        void *private_data
 
     cpdef enum ArrowType:
         NANOARROW_TYPE_BOOL
@@ -102,6 +110,8 @@ cdef class ArrowArrayImpl:
                                   array.array values) except -1
     cdef int append_vector(self, array.array value) except -1
     cdef int finish_building(self) except -1
+    cdef int populate_from_array(self, ArrowSchema* schema,
+                                 ArrowArray* array) except -1
     cdef int populate_from_metadata(self, ArrowType arrow_type, str name,
                                     int8_t precision, int8_t scale,
                                     ArrowTimeUnit time_unit,

src/oracledb/dataframe.py

@@ -32,6 +32,7 @@
 from typing import List
 
 from .arrow_array import ArrowArray
+from .arrow_impl import DataFrameImpl
 from . import errors
 
 
@@ -41,16 +42,37 @@ class DataFrame:
     def __init__(self):
         errors._raise_err(errors.ERR_INTERNAL_CREATION_REQUIRED)
 
+    @classmethod
+    def _from_arrow(cls, obj):
+        df = cls.__new__(cls)
+        df._initialize(DataFrameImpl.from_arrow_stream(obj))
+        return df
+
     @classmethod
     def _from_impl(cls, impl):
         df = cls.__new__(cls)
-        df._impl = impl
-        df._arrays = [ArrowArray._from_impl(a) for a in impl.get_arrays()]
-        df._arrays_by_name = {}
-        for array in df._arrays:
-            df._arrays_by_name[array.name] = array
+        df._initialize(impl)
         return df
 
+    def _initialize(self, impl):
+        """
+        Initializes the object given the implementation.
+        """
+        self._impl = impl
+        self._arrays = [ArrowArray._from_impl(a) for a in impl.get_arrays()]
+        self._arrays_by_name = {}
+        for array in self._arrays:
+            self._arrays_by_name[array.name] = array
+
+    def __arrow_c_stream__(self, requested_schema=None):
+        """
+        Returns the ArrowArrayStream PyCapsule which allows direct conversion
+        to foreign data frames that support this interface.
+        """
+        if requested_schema is not None:
+            raise NotImplementedError("requested_schema")
+        return self._impl.get_stream_capsule()
+
     def column_arrays(self) -> List:
         """
         Returns a list of the Arrow arrays corresponding to each column in the