An error occurs when using the GPU engines with the streaming executor #25592
Description
Checks
- I have checked that this issue has not already been reported.
- I have confirmed this bug exists on the latest version of Polars.
Reproducible example
import polars as pl
ctx = pl.SQLContext()
t0 = pl.LazyFrame(
{
"c0": [1],
}
)
ctx.register("t0", t0)
t1 = pl.LazyFrame(
{
"c0": [2],
}
)
ctx.register("t1", t1)
result = pl.concat([t0.join(t1, how='cross'), t0.with_columns(pl.lit(None))]).select(pl.len())
print("CPU Result:")
print(result.collect(engine="in-memory"))
print("CPU Streaming Result:")
print(result.collect(engine="streaming"))
print("GPU Result:")
print(result.collect(engine=pl.GPUEngine(raise_on_fail=True)))
print("GPU Streaming Result:")
print(result.collect(engine=pl.GPUEngine(raise_on_fail=True, executor="streaming", executor_options={"fallback_mode": "raise"})))Log output
CPU Result:
shape: (1, 1)
┌─────┐
│ len │
│ --- │
│ u32 │
╞═════╡
│ 2 │
└─────┘
CPU Streaming Result:
shape: (1, 1)
┌─────┐
│ len │
│ --- │
│ u32 │
╞═════╡
│ 2 │
└─────┘
GPU Result:
/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/utils.py:103: UserWarning: This selection is not supported for multiple partitions.
_fallback_inform(msg, rec.state["config_options"])
shape: (1, 1)
┌─────┐
│ len │
│ --- │
│ u32 │
╞═════╡
│ 2 │
└─────┘
GPU Streaming Result:
Traceback (most recent call last):
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/dsl/traversal.py", line 222, in __call__
return self.cache[value]
KeyError: Select({'len': <DataType(polars=UInt32, plc=<type_id.UINT32: 7>)>}, (NamedExpr(len, Cast(<DataType(polars=UInt32, plc=<type_id.UINT32: 7>)>, Len(<DataType(polars=UInt32, plc=<type_id.UINT32: 7>)>))),), True, Union({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, None, Projection({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, Join({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>, 'c0_right': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, (), (), ('Cross', False, None, '_right', False, 'none'), Cache({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, 211345494129781889079209974248998738334, None, Projection({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, DataFrameScan({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, <builtins.PyDataFrame object at 0x7081caaf53e0>, ('c0',)))), DataFrameScan({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, <builtins.PyDataFrame object at 0x70812b3da330>, ('c0',)))), Cache({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, 211345494129781889079209974248998738334, None, Projection({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, DataFrameScan({'c0': <DataType(polars=Int64, plc=<type_id.INT64: 4>)>}, <builtins.PyDataFrame object at 0x7081caaf53e0>, ('c0',))))))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/dsl/traversal.py", line 222, in __call__
return self.cache[value]
KeyError: Cast(<DataType(polars=UInt32, plc=<type_id.UINT32: 7>)>, Len(<DataType(polars=UInt32, plc=<type_id.UINT32: 7>)>))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/dsl/traversal.py", line 222, in __call__
return self.cache[value]
KeyError: Len(<DataType(polars=UInt32, plc=<type_id.UINT32: 7>)>)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/select.py", line 337, in _
return decompose_select(
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/select.py", line 87, in decompose_select
new_ne, partial_input_ir, _partition_info = decompose_expr_graph(
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/expressions.py", line 601, in decompose_expr_graph
expr, input_ir, partition_info = mapper(named_expr.value)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/dsl/traversal.py", line 224, in __call__
return self.cache.setdefault(value, self.fn(value, self))
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/expressions.py", line 496, in _decompose
children, input_irs, _partition_info = zip(
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/expressions.py", line 497, in <genexpr>
*(rec(c) for c in expr.children), strict=True
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/dsl/traversal.py", line 224, in __call__
return self.cache.setdefault(value, self.fn(value, self))
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/expressions.py", line 485, in _decompose
return _decompose_expr_node(
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/expressions.py", line 473, in _decompose_expr_node
raise NotImplementedError(
NotImplementedError: <class 'cudf_polars.dsl.expressions.unary.Len'> not supported for multiple partitions.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/xxx/bug.py", line 37, in <module>
print(result.collect(engine=pl.GPUEngine(raise_on_fail=True, executor="streaming", executor_options={"fallback_mode": "raise"})))
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/polars/_utils/deprecation.py", line 97, in wrapper
return function(*args, **kwargs)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/polars/lazyframe/opt_flags.py", line 328, in wrapper
return function(*args, **kwargs)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/polars/lazyframe/frame.py", line 2415, in collect
return wrap_df(ldf.collect(engine, callback))
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/polars/_utils/scan.py", line 27, in _execute_from_rust
return function(with_columns, *args)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/callback.py", line 263, in _callback
return evaluate_streaming(ir, config_options)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/parallel.py", line 283, in evaluate_streaming
ir, partition_info, _ = lower_ir_graph(ir, config_options)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/parallel.py", line 97, in lower_ir_graph
return *mapper(ir), state["stats"]
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/dsl/traversal.py", line 224, in __call__
return self.cache.setdefault(value, self.fn(value, self))
File "/xxx/anaconda3/envs/polars/lib/python3.10/functools.py", line 889, in wrapper
return dispatch(args[0].__class__)(*args, **kw)
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/select.py", line 345, in _
return _lower_ir_fallback(
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/utils.py", line 103, in _lower_ir_fallback
_fallback_inform(msg, rec.state["config_options"])
File "/xxx/anaconda3/envs/polars/lib/python3.10/site-packages/cudf_polars/experimental/utils.py", line 45, in _fallback_inform
raise NotImplementedError(msg)
NotImplementedError: This selection is not supported for multiple partitions.Issue description
A NotImplementedError will occur when using the GPU engines with the streaming executor.
However it will run normally when using the GPU engines with the in-memory executor.
Expected behavior
No error will occur when using the GPU engines with the streaming executor.
The output of GPU engines with the streaming executor is consistent with the CPU and GPU engines.
Installed versions
--------Version info---------
Polars: 1.34.0
Index type: UInt32
Platform: Linux-6.14.0-35-generic-x86_64-with-glibc2.39
Python: 3.10.19 (main, Oct 21 2025, 16:43:05) [GCC 11.2.0]
Runtime: rt32
----Optional dependencies----
Azure CLI <not installed>
adbc_driver_manager <not installed>
altair <not installed>
azure.identity <not installed>
boto3 <not installed>
cloudpickle <not installed>
connectorx <not installed>
deltalake <not installed>
fastexcel <not installed>
fsspec 2025.10.0
gevent <not installed>
google.auth <not installed>
great_tables <not installed>
matplotlib <not installed>
numpy 1.26.4
openpyxl <not installed>
pandas 2.3.3
polars_cloud <not installed>
pyarrow <not installed>
pydantic <not installed>
pyiceberg <not installed>
sqlalchemy <not installed>
torch <not installed>
xlsx2csv <not installed>
xlsxwriter <not installed>
cuda-bindings 12.9.4
cuda-pathfinder 1.3.2
cuda-python 12.9.4
cudf-polars 26.2.0a104
nvidia-ml-py 13.580.82
nvtx 0.2.13