Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Features
* Display of inline plots returned from `.|` operations.
* Don't attempt inline plots in the Windows console.
* Save plots, like parquets, with the `.>` operator.
* Display many native Python types returned from `.|` operations.


Bug Fixes
Expand Down
9 changes: 6 additions & 3 deletions doc/transforms.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,9 +53,12 @@ SELECT customer_id, COUNT(1) AS len FROM orders GROUP BY customer_id;
```

Transform expressions run with normal Python privileges, and expressions
should not be run from untrusted sources. If the transform operation
returns a Polars `DataFrame` or `Series`, the result is rendered by mycli
as tabular output. Most other return types will be silently ignored.
should not be run from untrusted sources.

If the transform operation returns a Polars `DataFrame` or `Series`, or a
Python type which can be rendered as a table, the result is rendered by mycli
as tabular output. `None` return values will be silently ignored, and other
return types will result in a warning message.

Transform expressions are useful for operations such as medians which
cannot be done (or are awkward) in SQL. Example:
Expand Down
59 changes: 51 additions & 8 deletions mycli/packages/polars_transform.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@

import builtins
from dataclasses import dataclass
import datetime
import decimal
import fractions
from io import BytesIO
from types import CodeType
from typing import Any, Iterable
from typing import Any, Iterable, Sequence

import sqlglot

Expand Down Expand Up @@ -36,6 +39,25 @@ class PolarsTransform:
altair: Any | None


def _is_coerceable_scalarish(value: Any) -> bool:
# pretend str/bytes are scalars
return isinstance(
value,
(
bool,
int,
float,
complex,
bytes,
str,
datetime.datetime,
datetime.timedelta,
decimal.Decimal,
fractions.Fraction,
),
)


def parse_polars_transform(command: str) -> PolarsPipeline | None:
"""Parse a SQL statement with optional Polars transform and file output."""
try:
Expand Down Expand Up @@ -207,10 +229,32 @@ def run_polars_transform(
try:
value = eval(
transform.code,
{'__builtins__': builtins, 'df': dataframe, 'pl': transform.polars, 'alt': transform.altair},
{
'__builtins__': builtins,
'df': dataframe,
'pl': transform.polars,
'alt': transform.altair,
},
)
except Exception as exc:
raise PolarsTransformError(f'Polars expression failed: {type(exc).__name__}: {exc}') from exc

if _is_coerceable_scalarish(value):
try:
value = transform.polars.DataFrame([value], schema=[str(value)])
except Exception as exc:
raise PolarsTransformError(f'Unable to render scalar as DataFrame: {type(exc).__name__}: {exc}') from exc
elif isinstance(value, dict):
try:
value = transform.polars.DataFrame(value)
except Exception as exc:
raise PolarsTransformError(f'Unable to render dictionary as DataFrame: {type(exc).__name__}: {exc}') from exc
elif isinstance(value, Sequence):
try:
value = transform.polars.Series(value, strict=False)
except Exception as exc:
raise PolarsTransformError(f'Unable to render Sequence as Series: {type(exc).__name__}: {exc}') from exc

if isinstance(value, transform.polars.DataFrame):
if output_path is not None:
if not output_path.lower().endswith('.parquet'):
Expand All @@ -221,7 +265,7 @@ def run_polars_transform(
raise PolarsTransformError(f'Unable to write Parquet file "{output_path}": {type(exc).__name__}: {exc}') from exc
return SQLResult(status=f'Wrote {len(value)} rows to {output_path}.')
return SQLResult(header=list(value.columns), rows=list(value.iter_rows()))
if isinstance(value, transform.polars.Series):
elif isinstance(value, transform.polars.Series):
column_name = value.name or 'value'
if output_path is not None:
if not output_path.lower().endswith('.parquet'):
Expand All @@ -233,7 +277,7 @@ def run_polars_transform(
raise PolarsTransformError(f'Unable to write Parquet file "{output_path}": {type(exc).__name__}: {exc}') from exc
return SQLResult(status=f'Wrote {len(series_dataframe)} rows to {output_path}.')
return SQLResult(header=[column_name], rows=[(item,) for item in value])
if transform.altair is not None and isinstance(value, transform.altair.TopLevelMixin):
elif transform.altair is not None and isinstance(value, transform.altair.TopLevelMixin):
if output_path is not None and not output_path.lower().endswith('.png'):
raise PolarsTransformError('Altair charts can only be written to ".png" files.')
if output_path is None and image_protocol == 'none':
Expand Down Expand Up @@ -265,8 +309,7 @@ def run_polars_transform(
except Exception as exc:
raise PolarsTransformError(f'Unable to render Altair chart: {type(exc).__name__}: {exc}') from exc
return SQLResult(image=png.getvalue(), image_protocol=image_protocol)
if output_path is not None:
raise PolarsTransformError(
'Polars transforms must return a DataFrame or Series for Parquet output, or an Altair chart for PNG output.'
)
elif value is None:
return SQLResult()

return SQLResult(status=f'Nothing could be displayed for return type: {type(value)}')
Loading
Loading