Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 20 additions & 5 deletions dataframely/experimental/infer_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,19 +123,34 @@ def _make_valid_identifier(name: str, col_index: int) -> str:
return valid_identifier


def _string_literal(value: str) -> str:
"""Render a string as a safely escaped Python string literal.

Double quotes are preferred for consistency with the generated code style, but
the escaping itself is delegated to :func:`repr` to guarantee that arbitrary
(potentially untrusted) input cannot break out of the literal.
"""
literal = repr(value)
if literal.startswith("'") and '"' not in value and "'" not in value:
# `repr` only escapes the quote character it uses as delimiter. If the value
# contains no quotes at all, the inner part can be re-delimited verbatim.
return f'"{literal[1:-1]}"'
return literal


def _get_dtype_args(dtype: pl.DataType, series: pl.Series) -> list[str]:
"""Get extra arguments for parameterized types."""
if isinstance(dtype, pl.Datetime):
args = []
if dtype.time_zone is not None:
args.append(f'time_zone="{dtype.time_zone}"')
args.append(f"time_zone={_string_literal(dtype.time_zone)}")
if dtype.time_unit != "us":
args.append(f'time_unit="{dtype.time_unit}"')
args.append(f"time_unit={_string_literal(dtype.time_unit)}")
return args

if isinstance(dtype, pl.Duration):
if dtype.time_unit != "us": # us is the default
return [f'time_unit="{dtype.time_unit}"']
return [f"time_unit={_string_literal(dtype.time_unit)}"]

if isinstance(dtype, pl.Decimal):
args = []
Expand Down Expand Up @@ -165,7 +180,7 @@ def _get_dtype_args(dtype: pl.DataType, series: pl.Series) -> list[str]:
fields_parts = []
for field in dtype.fields:
field_code = _dtype_to_column_code(series.struct.field(field.name))
fields_parts.append(f'"{field.name}": {field_code}')
fields_parts.append(f"{_string_literal(field.name)}: {field_code}")
return ["{" + ", ".join(fields_parts) + "}"]

return []
Expand All @@ -177,7 +192,7 @@ def _format_args(*args: str, nullable: bool = False, alias: str | None = None) -
if nullable:
all_args.append("nullable=True")
if alias is not None:
all_args.append(f'alias="{alias}"')
all_args.append(f"alias={_string_literal(alias)}")
return ", ".join(all_args)


Expand Down
42 changes: 42 additions & 0 deletions tests/experimental/test_infer_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,18 @@ class ConflictSchema(dy.Schema):
# Nullable inner types
pl.DataFrame({"list_with_nulls": [["a"], [None]]}),
pl.DataFrame({"struct_with_nulls": [{"key": "value"}, {"key": None}]}),
# Names requiring escaping
pl.DataFrame(
{
'a"b': ["test"],
"a'b": ["test"],
"a'\"b": ["test"],
"a\\b": ["test"],
"a\nb": ["test"],
"a\\": ["test"],
}
),
pl.DataFrame({"s": [{'a"b': "x", "a\\nb\n": "y"}]}),
],
)
def test_inferred_schema_validates_dataframe(df: pl.DataFrame) -> None:
Expand All @@ -401,3 +413,33 @@ def test_inferred_schema_validates_dataframe(df: pl.DataFrame) -> None:
exec(code, namespace) # noqa: S102
schema = namespace["TestSchema"]
assert schema.is_valid(df)


_INJECTION_PAYLOAD = 'x")\n INJECTED = True #'


@pytest.mark.parametrize(
"df",
[
pl.DataFrame({_INJECTION_PAYLOAD: ["test"]}),
pl.DataFrame({"s": [{_INJECTION_PAYLOAD: "test"}]}),
],
)
def test_no_code_injection_via_names(df: pl.DataFrame) -> None:
code = infer_schema(df, "TestSchema")
namespace: dict = {"dy": dy}
exec(code, namespace) # noqa: S102
schema = namespace["TestSchema"]
assert not hasattr(schema, "INJECTED")
assert schema.is_valid(df)


def test_escaped_alias_formatting() -> None:
df = pl.DataFrame({'a"b': ["test"], "a'b": ["test"], "a\nb": ["test"]})
result = infer_schema(df, schema_name="EscapeSchema")
expected = textwrap.dedent("""\
class EscapeSchema(dy.Schema):
a_b = dy.String(alias='a"b')
a_b_1 = dy.String(alias="a'b")
a_b_2 = dy.String(alias="a\\nb")""")
assert result == expected
Comment thread
AndreasAlbertQC marked this conversation as resolved.
Resolved
Loading