Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 33 additions & 22 deletions datafaker/db_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -161,23 +161,32 @@ def create_db_engine(
logger.error("DSN %s is malformed: %s", db_dsn, exc)
raise Exit(1) from exc

settings = {}
settings: list[str] = []
dialect_name = get_sync_engine(engine).dialect.name
if schema_name is not None:
if get_sync_engine(engine).dialect.name == "mssql":
engine = engine.execution_options(schema_translate_map={None: schema_name})
else:
settings["search_path"] = schema_name
match dialect_name:
case "mssql":
engine = engine.execution_options(
schema_translate_map={None: schema_name}
)
case "snowflake":
settings.append(f"USE SCHEMA {schema_name};")
case _:
settings.append(f"SET search_path TO {schema_name};")
dataset_views: list[str] = []
if parquet_dir is not None:
tables = find_parquet_tables(parquet_dir)
joined = ",".join(
sorted({str(t.root.parent) for t in tables if not t.is_dataset})
)
# double up single quotes
dj = joined.replace("'", "''")
# enclose in single quotes
settings["file_search_path"] = f"'{dj}'"
dataset_views = [_dataset_view_sql(t) for t in tables if t.is_dataset]
match dialect_name:
case "duckdb":
tables = find_parquet_tables(parquet_dir)
# double up single quotes
dj = str(parquet_dir).replace("'", "''")
settings.append(f"SET file_search_path TO '{dj}';")
dataset_views = [_dataset_view_sql(t) for t in tables if t.hive_files()]
case _:
tables = []
logger.error(
"--parquet-dir cannot be used with %s, only duckdb.", dialect_name
)

if settings or dataset_views:
event_engine = get_sync_engine(engine)
Expand Down Expand Up @@ -219,11 +228,15 @@ def create_db_engine_dst(
return create_db_engine(db_dsn, schema_name, use_asyncio)


def get_metadata(engine: Engine, schema_name: Optional[str] = None) -> MetaData:
def get_metadata(
engine: Engine,
schema_name: Optional[str],
views_to_tables: bool,
) -> MetaData:
"""Get the MetaData object associated with the engine passed."""
md = MetaData()
try:
md.reflect(engine, schema=schema_name)
md.reflect(engine, schema=schema_name, views=views_to_tables)
except OperationalError as exc:
logger.error("Cannot connect to database: %s", exc)
raise Exit(1) from exc
Expand All @@ -237,7 +250,7 @@ def _sql_quote(text: str, quote: str) -> str:

def _dataset_view_sql(table: ParquetTable) -> str:
"""Make the SQL that exposes a directory of parquet files as one table."""
files = ",".join(_sql_quote(str(f), "'") for f in table.files)
files = ",".join(_sql_quote(str(f), "'") for f in table.hive_files())
return (
f"CREATE OR REPLACE TEMP VIEW {_sql_quote(table.name, chr(34))} AS "
f"SELECT * FROM read_parquet([{files}], "
Expand All @@ -259,16 +272,14 @@ def create_dataset_views(
cursor.close()


def set_db_settings(connection: DBAPIConnection, settings: Mapping[str, str]) -> None:
"""Set the SEARCH_PATH for a PostgreSQL connection."""
def set_db_settings(connection: DBAPIConnection, settings: list[str]) -> None:
"""Run commands."""
# https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#remote-schema-table-introspection-and-postgresql-search-path
existing_autocommit = connection.autocommit
connection.autocommit = True

cursor = connection.cursor()
# Parametrised queries don't work with asyncpg, hence the f-string.
sql = "".join(f"SET {k} TO {v};" for k, v in settings.items())
cursor.execute(sql)
cursor.execute("\n".join(settings))
cursor.close()

connection.autocommit = existing_autocommit
Expand Down
17 changes: 17 additions & 0 deletions datafaker/dialects.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from collections.abc import Mapping
from typing import Any, Optional, TypeVar

import snowflake.sqlalchemy.custom_types as snowflake_types
from sqlalchemy import Column, Select, Table
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.schema import CreateSchema, CreateTable
Expand Down Expand Up @@ -515,3 +516,19 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str:
same random number gets chosen for each row.
"""
return "NEWID()"


@compiles(snowflake_types.TIMESTAMP_LTZ)
def compile_timestampltz_generic(
_element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any
) -> str:
"""Render TIMESTAMP_LTZ as DateTime."""
return compiler.process(DateTime(timezone=False), **kw)


@compiles(snowflake_types.TIMESTAMP_LTZ, "snowflake")
def compile_timestampltz_ddb(
element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any
) -> str:
"""Render TIMESTAMP_LTZ as TIMESTAMP_LTZ."""
return compiler.process(element, **kw)
13 changes: 7 additions & 6 deletions datafaker/interactive/generators.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from enum import Enum
from typing import Any, Callable, Optional, cast

from sqlalchemy import Column, and_, literal_column, select
from sqlalchemy import Column, and_, select
from sqlalchemy.types import Integer

from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private
Expand Down Expand Up @@ -1032,12 +1032,13 @@ def _print_select_aggregate_query(self, table_name: str, prop: Proposer) -> None
def _get_column_data(
self, count: int, to_str: Callable[[Any], str] = repr
) -> list[list[str]]:
columns = self._get_column_names()
col_exprs = [literal_column(col) for col in columns]
nonnull_clauses = [literal_column(col).isnot(None) for col in columns]
column_names = self._get_column_names()
table = self.table_metadata()
columns = [table.columns[c] for c in column_names]
nonnull_clauses = [col.isnot(None) for col in columns]
stmt = (
select(*col_exprs)
.select_from(self.table_metadata())
select(*columns)
.select_from(table)
.where(and_(*nonnull_clauses))
.order_by(Random())
.limit(count)
Expand Down
2 changes: 1 addition & 1 deletion datafaker/interactive/missingness.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ def _copy_entries(self) -> None:
"kwargs": {
"patterns": f'SRC_STATS["{src_stat_key}"]["results"]'
},
"columns": [c.name for c in entry.new_type.columns],
"columns": [str(c.name) for c in entry.new_type.columns],
}
]
src_stats.append(
Expand Down
21 changes: 8 additions & 13 deletions datafaker/interactive/table.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from dataclasses import dataclass
from typing import Any, cast

from sqlalchemy import func, literal_column, select
from sqlalchemy import func, select

from datafaker.dialects import Random
from datafaker.interactive.base import (
Expand Down Expand Up @@ -469,26 +469,21 @@ def complete_data(
return []
return self.get_column_completions(text)

def print_column_data(self, column: str, count: int, min_length: int) -> None:
def print_column_data(self, column_name: str, count: int, min_length: int) -> None:
"""
Print a sample of data from a certain column of the current table.

:param column: The name of the column to report on.
:param column_name: The name of the column to report on.
:param count: The number of rows to sample.
:param min_length: The minimum length of text to choose from (0 for any text).
"""
col_expr = literal_column(column)
table = self.table_metadata()
column = table.columns[column_name]
if 0 < min_length:
where_clause = func.length(col_expr) >= min_length
where_clause = func.length(column) >= min_length
else:
where_clause = col_expr.isnot(None)
stmt = (
select(col_expr)
.select_from(self.table_metadata())
.where(where_clause)
.order_by(Random())
.limit(count)
)
where_clause = column.isnot(None)
stmt = select(column).where(where_clause).order_by(Random()).limit(count)
with self.sync_engine.connect() as connection:
result = connection.execute(stmt)
self.columnize([str(x[0]) for x in result.all()])
Expand Down
9 changes: 8 additions & 1 deletion datafaker/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,11 +172,17 @@ def make_tables(
help=(
"Directory of Parquet files to consider part of the database."
" This can be useful when using DuckDB."
" The .parquet files or directories containing partitioned tables"
" must all be directly inside this named directory."
" Make sure you check the output!"
),
file_okay=False,
dir_okay=True,
),
views_to_tables: bool = Option(
False,
help=("Generate all views as tables in the synthetic version."),
),
) -> None:
"""Make a YAML file representing the tables in the schema.

Expand All @@ -192,7 +198,8 @@ def make_tables(
content = make_tables_file(
get_source_dsn(),
get_source_schema(),
parquet_dir,
views_to_tables=views_to_tables,
parquet_dir=parquet_dir,
)
orm_file_path.write_text(content, encoding="utf-8")
logger.debug("%s created.", orm_file)
Expand Down
29 changes: 26 additions & 3 deletions datafaker/make.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
from typing import Any, Final, Optional, Tuple, Type

import pandas as pd
import snowflake.sqlalchemy.custom_types as snowflake_types
import snsql
import sqlalchemy
import typer
import yaml
from mimesis.providers.base import BaseProvider
Expand Down Expand Up @@ -39,7 +41,7 @@
get_sync_engine,
get_vocabulary_table_names,
)
from datafaker.parquet2orm import get_parquet_orm
from datafaker.parquet2orm import add_guessed_keys, get_initial_parquet_orm
from datafaker.settings import get_source_dsn, get_source_schema
from datafaker.utils import (
get_columns_assigned,
Expand All @@ -58,6 +60,8 @@
if issubclass(entry, BaseProvider) and entry.__module__ == "datafaker.providers":
PROVIDER_IMPORTS.append(entry_name)

DEFAULT_UNIQUE_CONSTRAINT_TRIES: int | None = 500


@dataclass
class VocabularyTableGeneratorInfo:
Expand Down Expand Up @@ -378,6 +382,10 @@ def get_result_mappings(
generator=_string_generator,
choice=True,
),
snowflake_types.TIMESTAMP_LTZ: GeneratorInfo(
generator="generic.datetime.datetime",
arg_types={"start": int, "end": int},
),
}


Expand Down Expand Up @@ -654,6 +662,8 @@ def get_generation_info(
max_unique_constraint_tries = get_property_or_none(
config, "max-unique-constraint-tries", int
)
if max_unique_constraint_tries is None:
max_unique_constraint_tries = DEFAULT_UNIQUE_CONSTRAINT_TRIES
return GenerationInfo(
provider_imports=PROVIDER_IMPORTS,
row_generator_module_name=row_generator_module_name,
Expand Down Expand Up @@ -694,26 +704,39 @@ def _generate_vocabulary_table(
download_table(table, engine, yaml_file_name, compress)


# pylint: disable=too-many-arguments disable=too-many-positional-arguments
def make_tables_file(
db_dsn: str,
schema_name: Optional[str],
views_to_tables: bool,
parquet_dir: Optional[Path] = None,
engine: Optional[Engine] = None,
) -> str:
"""Construct the YAML file representing the schema."""
if engine is None:
engine = get_sync_engine(create_db_engine(db_dsn, schema_name=schema_name))

metadata = get_metadata(engine, schema_name=schema_name)
metadata = get_metadata(
engine, schema_name=schema_name, views_to_tables=views_to_tables
)
meta_dict = metadata_to_dict(metadata, schema_name, engine, parquet_dir)
inspector = sqlalchemy.inspect(engine)
if hasattr(inspector, "get_view_names"):
view_names = set(inspector.get_view_names())
else:
view_names = set()

parquet_table_names = set()
if parquet_dir is not None:
extra_meta = get_parquet_orm(parquet_dir)
extra_meta = get_initial_parquet_orm(parquet_dir)
if extra_meta:
parquet_table_names = set(extra_meta.keys())
md_tables: dict[str, Any] = get_property(meta_dict, "tables", {})
new_tables = {**extra_meta, **md_tables}
meta_dict["tables"] = new_tables

add_guessed_keys(meta_dict["tables"], view_names | parquet_table_names)

return yaml.dump(meta_dict, sort_keys=False)


Expand Down
Loading
Loading