Fix Polars expr pushdown (#102)

See #98 

We have our own Polars IO plugin to create lazy polars dataframes. We
try to push as many predicates down into DuckDB as we can, for which we
try to map Polars expressions (including datatypes) to SQL expressions.
To do this we depend on Polar's `Expr.meta.serialize`.

None of this is very stable. Polars IO source plugins are marked
`@unstable` and `Expr.meta.serialize` says "Serialization is not stable
across Polars versions".

In this case the problems seems to come from Polars requiring an
explicit scale to be set for decimals ([this
pr](pola-rs/polars#24542)). The serialized
format seems to have changed into:
```json
{
  "expr": {
    "Literal": {
      "Scalar": {
        "Decimal": [
          1,
          38, // This now includes the scale
          0
        ]
      }
    }
  },
  "dtype": {
    "Literal": {
      "Decimal": [
        38, // this was already there
        0
      ]
    }
  },
  "options": "Strict"
}

```

Interestingly, even if we explicitly set precision to e.g. 20, the
relevant part of the serialized expression looks as follows:
```json
{
  "expr": {
    "Literal": {
      "Scalar": {
        "Decimal": [
          1,
          38, // Still 38?
          0
        ]
      }
    }
  },
  "dtype": {
    "Literal": {
      "Decimal": [
        20, // This is correct
        0
      ]
    }
  },
  "options": "Strict"
}
```

This PR allows for both a 2 and 3 item list for decimals.

Author: evertlammerts

duckdb/polars_io.py

@@ -1,5 +1,6 @@
 from __future__ import annotations  # noqa: D100
 
+import contextlib
 import datetime
 import json
 import typing
@@ -176,9 +177,12 @@ def _pl_tree_to_sql(tree: _ExpressionTree) -> str:
         if dtype.startswith("{'Decimal'") or dtype == "Decimal":
             decimal_value = value["Decimal"]
             assert isinstance(decimal_value, list), (
-                f"A {dtype} should be a two member list but got {type(decimal_value)}"
+                f"A {dtype} should be a two or three member list but got {type(decimal_value)}"
             )
-            return str(Decimal(decimal_value[0]) / Decimal(10 ** decimal_value[1]))
+            assert 2 <= len(decimal_value) <= 3, (
+                f"A {dtype} should be a two or three member list but got {len(decimal_value)} member list"
+            )
+            return str(Decimal(decimal_value[0]) / Decimal(10 ** decimal_value[-1]))
 
         # Datetime with microseconds since epoch
         if dtype.startswith("{'Datetime'") or dtype == "Datetime":
@@ -260,7 +264,8 @@ def source_generator(
             relation_final = relation_final.limit(n_rows)
         if predicate is not None:
             # We have a predicate, if possible, we push it down to DuckDB
-            duck_predicate = _predicate_to_expression(predicate)
+            with contextlib.suppress(AssertionError, KeyError):
+                duck_predicate = _predicate_to_expression(predicate)
         # Try to pushdown filter, if one exists
         if duck_predicate is not None:
             relation_final = relation_final.filter(duck_predicate)

tests/fast/arrow/test_polars.py

@@ -1,4 +1,5 @@
 import datetime
+import json
 
 import pytest
 
@@ -8,7 +9,7 @@
 arrow = pytest.importorskip("pyarrow")
 pl_testing = pytest.importorskip("polars.testing")
 
-from duckdb.polars_io import _predicate_to_expression  # noqa: E402
+from duckdb.polars_io import _pl_tree_to_sql, _predicate_to_expression  # noqa: E402
 
 
 def valid_filter(filter):
@@ -175,7 +176,7 @@ def test_polars_column_with_tricky_name(self, duckdb_cursor):
             "UBIGINT",
             "FLOAT",
             "DOUBLE",
-            # "HUGEINT",
+            "HUGEINT",
             "DECIMAL(4,1)",
             "DECIMAL(9,1)",
             "DECIMAL(18,4)",
@@ -605,3 +606,50 @@ def test_polars_lazy_many_batches(self, duckdb_cursor):
         correct = duckdb_cursor.execute("FROM t").fetchall()
 
         assert res == correct
+
+    def test_invalid_expr_json(self):
+        bad_key_expr = """
+        {
+            "BinaryExpr": {
+                "left": { "Column": "foo" },
+                "middle": "Gt",
+                "right": { "Literal": { "Int": 5 } }
+            }
+        }
+        """
+        with pytest.raises(KeyError, match="'op'"):
+            _pl_tree_to_sql(json.loads(bad_key_expr))
+
+        bad_type_expr = """
+        {
+            "BinaryExpr": {
+                "left": { "Column": [ "foo" ]  },
+                "op": "Gt",
+                "right": { "Literal": { "Int": 5 } }
+            }
+        }
+        """
+        with pytest.raises(AssertionError, match="The col name of a Column should be a str but got"):
+            _pl_tree_to_sql(json.loads(bad_type_expr))
+
+    def test_decimal_scale(self):
+        scalar_decimal_no_scale = """
+          { "Scalar": {
+            "Decimal": [
+              1,
+              0
+            ]
+          } }
+        """
+        assert _pl_tree_to_sql(json.loads(scalar_decimal_no_scale)) == "1"
+
+        scalar_decimal_scale = """
+          { "Scalar": {
+            "Decimal": [
+              1,
+              38,
+              0
+            ]
+          } }
+        """
+        assert _pl_tree_to_sql(json.loads(scalar_decimal_scale)) == "1"