Upgrade to Datafusion 51 (#1311)

* Upgrade to Datafusion 51

* Fix clippy

* Upgrade to Datafusion 51

* Fix clippy

* Refactor test_arrow_c_stream_interrupted to handle exceptions in a separate thread

* Improve exception handling in wait_for_future

Updated wait_for_future to surface pending Python exceptions by
executing bytecode during signal checks, ensuring that asynchronous
interrupts are processed promptly. Enhanced PartitionedDataFrameStreamReader
to cancel remaining partition streams on projection errors or Python
interrupts, allowing for clean iteration stops. Added regression tests
to validate interrupted Arrow C stream reads and improve timing
for RecordBatchReader.read_all cancellations.

* Refactor signal checking in future collection to simplify error handling

* Simplify KeyboardInterrupt check in test_arrow_c_stream_interrupted

* rm test_record_batch_reader_interrupt_exits_quickly

* Refactor test_arrow_c_stream_interrupted to improve exception handling and readability

* Improve exception handling in test_arrow_c_stream_interrupted to check for KeyboardInterrupt

* Add comment - handle KeyboardInterrupt more effectively

* Remove unnecessary stream cancellation on error in PartitionedDataFrameStreamReader

* Add jupyter notebook for test

* Revert "Add jupyter notebook for test"

This reverts commit 784929d.

* Simplify error handling in PartitionedDataFrameStreamReader

* Fix ruff warnings

* Fix ruff warnings

* Update src/utils.rs

Co-authored-by: Tim Saucer <timsaucer@gmail.com>

* Fix clippy

---------

Co-authored-by: Siew Kam Onn <kosiew@gmail.com>
Co-authored-by: Tim Saucer <timsaucer@gmail.com>

Author: 3 people

Cargo.lock

@@ -48,20 +48,20 @@ tokio = { version = "1.47", features = [
     "rt-multi-thread",
     "sync",
 ] }
-pyo3 = { version = "0.25", features = [
+pyo3 = { version = "0.26", features = [
     "extension-module",
     "abi3",
     "abi3-py310",
 ] }
-pyo3-async-runtimes = { version = "0.25", features = ["tokio-runtime"] }
-pyo3-log = "0.12.4"
-arrow = { version = "56", features = ["pyarrow"] }
-arrow-select = { version = "56" }
-datafusion = { version = "50", features = ["avro", "unicode_expressions"] }
-datafusion-substrait = { version = "50", optional = true }
-datafusion-proto = { version = "50" }
-datafusion-ffi = { version = "50" }
-prost = "0.13.1" # keep in line with `datafusion-substrait`
+pyo3-async-runtimes = { version = "0.26", features = ["tokio-runtime"] }
+pyo3-log = "0.13.2"
+arrow = { version = "57", features = ["pyarrow"] }
+arrow-select = { version = "57" }
+datafusion = { version = "51", features = ["avro", "unicode_expressions"] }
+datafusion-substrait = { version = "51", optional = true }
+datafusion-proto = { version = "51" }
+datafusion-ffi = { version = "51" }
+prost = "0.14.1" # keep in line with `datafusion-substrait`
 uuid = { version = "1.18", features = ["v4"] }
 mimalloc = { version = "0.1", optional = true, default-features = false, features = [
     "local_dynamic_tls",
@@ -80,8 +80,8 @@ log = "0.4.27"
 parking_lot = "0.12"
 
 [build-dependencies]
-prost-types = "0.13.1"     # keep in line with `datafusion-substrait`
-pyo3-build-config = "0.25"
+prost-types = "0.14.1"     # keep in line with `datafusion-substrait`
+pyo3-build-config = "0.26"
 
 [lib]
 name = "datafusion_python"