Added support for all of the signed and unsigned fixed width integer

types when ingesting data frames supporting the Arrow PyCapsule
interface. Previously only int64 was supported.

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -37,6 +37,9 @@ Thick Mode Changes
 Common Changes
 ++++++++++++++
 
+#)  Added support for all of the signed and unsigned fixed width integer types
+    when ingesting data frames supporting the Arrow PyCapsule interface.
+    Previously only int64 was supported.
 #)  Added ``fetch_lobs`` and ``fetch_decimals`` parameters where applicable to
     the methods used for fetching rows or dataframes from the database. Note
     that for the creation of pipeline operations, if these parameters are not

src/oracledb/arrow_impl.pxd

@@ -68,6 +68,8 @@ cdef extern from "nanoarrow.h":
         NANOARROW_TYPE_FIXED_SIZE_LIST
         NANOARROW_TYPE_FLOAT
         NANOARROW_TYPE_INT8
+        NANOARROW_TYPE_INT16
+        NANOARROW_TYPE_INT32
         NANOARROW_TYPE_INT64
         NANOARROW_TYPE_LARGE_BINARY
         NANOARROW_TYPE_LARGE_STRING
@@ -77,7 +79,9 @@ cdef extern from "nanoarrow.h":
         NANOARROW_TYPE_STRUCT
         NANOARROW_TYPE_TIMESTAMP
         NANOARROW_TYPE_UINT8
+        NANOARROW_TYPE_UINT16
         NANOARROW_TYPE_UINT32
+        NANOARROW_TYPE_UINT64
         NANOARROW_TYPE_UNINITIALIZED
 
     cpdef enum ArrowTimeUnit:
@@ -139,10 +143,12 @@ cdef class ArrowArrayImpl:
                         double* value) except -1
     cdef int get_float(self, int64_t index, bint* is_null,
                        float* value) except -1
-    cdef int get_int64(self, int64_t index, bint* is_null,
-                       int64_t* value) except -1
+    cdef int get_int(self, ArrowType arrow_type, int64_t index, bint* is_null,
+                     int64_t* value) except -1
     cdef int get_length(self, int64_t* length) except -1
     cdef object get_sparse_vector(self, int64_t index, bint* is_null)
+    cdef int get_uint(self, ArrowType arrow_type, int64_t index, bint* is_null,
+                      uint64_t* value) except -1
     cdef object get_vector(self, int64_t index, bint* is_null)
     cdef int populate_from_array(self, ArrowSchema* schema,
                                  ArrowArray* array) except -1

src/oracledb/base_impl.pyx

@@ -41,6 +41,7 @@ from libc.stdint cimport UINT8_MAX, UINT16_MAX, UINT32_MAX, UINT64_MAX
 from libc.stdlib cimport atoi, atof
 from libc.string cimport memcpy
 from cpython cimport array
+from cpython.conversion cimport PyOS_snprintf
 
 from .constants import VECTOR_META_FLAG_SPARSE_VECTOR
 
@@ -59,6 +60,8 @@ from .arrow_impl cimport (
     NANOARROW_TYPE_FIXED_SIZE_LIST,
     NANOARROW_TYPE_FLOAT,
     NANOARROW_TYPE_INT8,
+    NANOARROW_TYPE_INT16,
+    NANOARROW_TYPE_INT32,
     NANOARROW_TYPE_INT64,
     NANOARROW_TYPE_LIST,
     NANOARROW_TYPE_LARGE_BINARY,
@@ -67,6 +70,9 @@ from .arrow_impl cimport (
     NANOARROW_TYPE_STRUCT,
     NANOARROW_TYPE_TIMESTAMP,
     NANOARROW_TYPE_UINT8,
+    NANOARROW_TYPE_UINT16,
+    NANOARROW_TYPE_UINT32,
+    NANOARROW_TYPE_UINT64,
     ArrowArrayImpl,
 )
 

src/oracledb/impl/arrow/array.pyx

@@ -351,16 +351,24 @@ cdef class ArrowArrayImpl:
             ptr = <float*> self.arrow_array.buffers[1]
             value[0] = ptr[index]
 
-    cdef int get_int64(self, int64_t index, bint* is_null,
-                       int64_t* value) except -1:
+    cdef int get_int(self, ArrowType arrow_type, int64_t index, bint* is_null,
+                     int64_t* value) except -1:
         """
-        Return an int64_t value at the specified index from the Arrow array.
+        Return an int64_t value at the specified index from the Arrow array
+        for all signed integer types.
         """
-        cdef int64_t* ptr
+        cdef const void* ptr
         self._get_is_null(index, is_null)
         if not is_null[0]:
-            ptr = <int64_t*> self.arrow_array.buffers[1]
-            value[0] = ptr[index]
+            ptr = self.arrow_array.buffers[1]
+            if arrow_type == NANOARROW_TYPE_INT8:
+                value[0] = (<int8_t*> ptr)[index]
+            elif arrow_type == NANOARROW_TYPE_INT16:
+                value[0] = (<int16_t*> ptr)[index]
+            elif arrow_type == NANOARROW_TYPE_INT32:
+                value[0] = (<int32_t*> ptr)[index]
+            else:
+                value[0] = (<int64_t*> ptr)[index]
 
     cdef int get_length(self, int64_t* length) except -1:
         """
@@ -415,6 +423,25 @@ cdef class ArrowArrayImpl:
                    num_elements * self.schema_impl.child_element_size)
             return (num_dimensions, indices, values)
 
+    cdef int get_uint(self, ArrowType arrow_type, int64_t index, bint* is_null,
+                      uint64_t* value) except -1:
+        """
+        Return a uint64_t value at the specified index from the Arrow array
+        for all unsigned integer types.
+        """
+        cdef const void* ptr
+        self._get_is_null(index, is_null)
+        if not is_null[0]:
+            ptr = self.arrow_array.buffers[1]
+            if arrow_type == NANOARROW_TYPE_UINT8:
+                value[0] = (<uint8_t*> ptr)[index]
+            elif arrow_type == NANOARROW_TYPE_UINT16:
+                value[0] = (<uint16_t*> ptr)[index]
+            elif arrow_type == NANOARROW_TYPE_UINT32:
+                value[0] = (<uint32_t*> ptr)[index]
+            else:
+                value[0] = (<uint64_t*> ptr)[index]
+
     cdef object get_vector(self, int64_t index, bint* is_null):
         """
         Return a vector value at the specified index from the Arrow array.

src/oracledb/impl/arrow/schema.pyx

@@ -146,10 +146,17 @@ cdef class ArrowSchemaImpl:
                 NANOARROW_TYPE_DOUBLE,
                 NANOARROW_TYPE_FIXED_SIZE_BINARY,
                 NANOARROW_TYPE_FLOAT,
+                NANOARROW_TYPE_INT8,
+                NANOARROW_TYPE_INT16,
+                NANOARROW_TYPE_INT32,
                 NANOARROW_TYPE_INT64,
                 NANOARROW_TYPE_LARGE_BINARY,
                 NANOARROW_TYPE_LARGE_STRING,
                 NANOARROW_TYPE_STRING,
+                NANOARROW_TYPE_UINT8,
+                NANOARROW_TYPE_UINT16,
+                NANOARROW_TYPE_UINT32,
+                NANOARROW_TYPE_UINT64,
         ) and not (
                 schema_view.type == NANOARROW_TYPE_STRUCT
                 and self._is_sparse_vector()

src/oracledb/impl/base/converters.pyx

@@ -37,18 +37,39 @@ cdef object convert_arrow_to_oracle_data(OracleMetadata metadata,
     Converts the value stored in Arrow format to an OracleData structure.
     """
     cdef:
-        int64_t int64_value, days, seconds, useconds
+        int64_t int_value, days, seconds, useconds
         SparseVectorImpl sparse_impl
         ArrowType arrow_type
+        uint64_t uint_value
         OracleRawBytes* rb
         tuple sparse_info
         bytes temp_bytes
+        ssize_t buf_len
+        char buf[21]
 
     arrow_type = metadata._schema_impl.arrow_type
-    if arrow_type == NANOARROW_TYPE_INT64:
-        array_impl.get_int64(array_index, &data.is_null, &int64_value)
+    if arrow_type in (
+        NANOARROW_TYPE_INT8,
+        NANOARROW_TYPE_INT16,
+        NANOARROW_TYPE_INT32,
+        NANOARROW_TYPE_INT64,
+    ):
+        array_impl.get_int(arrow_type, array_index, &data.is_null, &int_value)
+        if not data.is_null:
+            buf_len = PyOS_snprintf(buf, sizeof(buf), "%lld", int_value)
+            temp_bytes = buf[:buf_len]
+            convert_bytes_to_oracle_data(&data.buffer, temp_bytes)
+            return temp_bytes
+    elif arrow_type in (
+        NANOARROW_TYPE_UINT8,
+        NANOARROW_TYPE_UINT16,
+        NANOARROW_TYPE_UINT32,
+        NANOARROW_TYPE_UINT64,
+    ):
+        array_impl.get_uint(arrow_type, array_index, &data.is_null, &uint_value)
         if not data.is_null:
-            temp_bytes = str(int64_value).encode()
+            buf_len = PyOS_snprintf(buf, sizeof(buf), "%llu", uint_value)
+            temp_bytes = buf[:buf_len]
             convert_bytes_to_oracle_data(&data.buffer, temp_bytes)
             return temp_bytes
     elif arrow_type == NANOARROW_TYPE_DOUBLE:
@@ -70,10 +91,10 @@ cdef object convert_arrow_to_oracle_data(OracleMetadata metadata,
         array_impl.get_bytes(array_index, &data.is_null, <char**> &rb.ptr,
                               &rb.num_bytes)
     elif arrow_type == NANOARROW_TYPE_TIMESTAMP:
-        array_impl.get_int64(array_index, &data.is_null, &int64_value)
+        array_impl.get_int(arrow_type, array_index, &data.is_null, &int_value)
         if not data.is_null:
-            seconds = int64_value // array_impl.schema_impl.time_factor
-            useconds = int64_value % array_impl.schema_impl.time_factor
+            seconds = int_value // array_impl.schema_impl.time_factor
+            useconds = int_value % array_impl.schema_impl.time_factor
             days = seconds // (24 * 60 * 60)
             seconds = seconds % (24 * 60 * 60)
             if array_impl.schema_impl.time_factor == 1_000:

src/oracledb/impl/base/metadata.pyx

@@ -162,8 +162,17 @@ cdef class OracleMetadata:
         cdef:
             OracleMetadata metadata = OracleMetadata.__new__(OracleMetadata)
             ArrowType arrow_type = schema_impl.arrow_type
-        if arrow_type in (NANOARROW_TYPE_DECIMAL128,
-                          NANOARROW_TYPE_INT64):
+        if arrow_type in (
+            NANOARROW_TYPE_DECIMAL128,
+            NANOARROW_TYPE_INT8,
+            NANOARROW_TYPE_INT16,
+            NANOARROW_TYPE_INT32,
+            NANOARROW_TYPE_INT64,
+            NANOARROW_TYPE_UINT8,
+            NANOARROW_TYPE_UINT16,
+            NANOARROW_TYPE_UINT32,
+            NANOARROW_TYPE_UINT64,
+        ):
             metadata.dbtype = DB_TYPE_NUMBER
         elif arrow_type == NANOARROW_TYPE_STRING:
             metadata.dbtype = DB_TYPE_VARCHAR

tests/sql/create_schema.sql

@@ -410,6 +410,7 @@ create table &main_user..TestDataframe (
     CreditScore                         number(3, 0),
     LastUpdated                         timestamp,
     DecimalData                         number(15, 4),
+    IntegerData                         number(20),
     FloatData                           binary_float,
     DoubleData                          binary_double,
     RawData                             raw(100),

tests/test_8900_dataframe_ingestion.py

@@ -859,6 +859,45 @@ def test_8918(self):
         fetched_df = pyarrow.table(odf)
         self.assertTrue(fetched_df.equals(df))
 
+    def test_8919(self):
+        "8919 - test ingestion with various integer data types"
+        scenarios = [
+            ([-(2**7), 0, 2**7 - 1], pyarrow.int8()),
+            ([-(2**15), 0, 2**15 - 1], pyarrow.int16()),
+            ([-(2**31), 0, 2**31 - 1], pyarrow.int32()),
+            ([-(2**63), 0, 2**63 - 1], pyarrow.int64()),
+            ([0, 2**7, 2**8 - 1], pyarrow.uint8()),
+            ([0, 2**15, 2**16 - 1], pyarrow.uint16()),
+            ([0, 2**31, 2**32 - 1], pyarrow.uint32()),
+            ([0, 2**63, 2**64 - 1], pyarrow.uint64()),
+        ]
+        names = ["Id", "IntegerData"]
+        for values, dtype in scenarios:
+            with self.subTest(dtype=str(dtype)):
+                arrays = [
+                    pyarrow.array([1, 2, 3], pyarrow.int8()),
+                    pyarrow.array(values, dtype),
+                ]
+                df = pyarrow.table(arrays, names)
+                self.cursor.execute("delete from TestDataFrame")
+                self.cursor.executemany(
+                    """
+                    insert into TestDataFrame (Id, IntegerData)
+                    values (:1, :2)
+                    """,
+                    df,
+                )
+                self.conn.commit()
+                self.cursor.execute(
+                    """
+                    select to_char(IntegerData)
+                    from TestDataFrame
+                    order by Id
+                    """
+                )
+                fetched_values = [int(s) for s, in self.cursor]
+                self.assertEqual(fetched_values, values)
+
 
 if __name__ == "__main__":
     test_env.run_test_cases()

tests/test_9000_dataframe_ingestion_async.py

@@ -860,6 +860,45 @@ async def test_9018(self):
         fetched_df = pyarrow.table(odf)
         self.assertTrue(fetched_df.equals(df))
 
+    async def test_9019(self):
+        "9019 - test ingestion with various integer data types"
+        scenarios = [
+            ([-(2**7), 0, 2**7 - 1], pyarrow.int8()),
+            ([-(2**15), 0, 2**15 - 1], pyarrow.int16()),
+            ([-(2**31), 0, 2**31 - 1], pyarrow.int32()),
+            ([-(2**63), 0, 2**63 - 1], pyarrow.int64()),
+            ([0, 2**7, 2**8 - 1], pyarrow.uint8()),
+            ([0, 2**15, 2**16 - 1], pyarrow.uint16()),
+            ([0, 2**31, 2**32 - 1], pyarrow.uint32()),
+            ([0, 2**63, 2**64 - 1], pyarrow.uint64()),
+        ]
+        names = ["Id", "IntegerData"]
+        for values, dtype in scenarios:
+            with self.subTest(dtype=str(dtype)):
+                arrays = [
+                    pyarrow.array([1, 2, 3], pyarrow.int8()),
+                    pyarrow.array(values, dtype),
+                ]
+                df = pyarrow.table(arrays, names)
+                await self.cursor.execute("delete from TestDataFrame")
+                await self.cursor.executemany(
+                    """
+                    insert into TestDataFrame (Id, IntegerData)
+                    values (:1, :2)
+                    """,
+                    df,
+                )
+                await self.conn.commit()
+                await self.cursor.execute(
+                    """
+                    select to_char(IntegerData)
+                    from TestDataFrame
+                    order by Id
+                    """
+                )
+                fetched_values = [int(s) async for s, in self.cursor]
+                self.assertEqual(fetched_values, values)
+
 
 if __name__ == "__main__":
     test_env.run_test_cases()