Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions datafaker/dialects.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Dialect differences."""

import re
from collections.abc import Mapping
from typing import Any, Optional, TypeVar
Expand Down Expand Up @@ -518,6 +519,15 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str:
return "NEWID()"


def random_sample(query: Select, n: int | ColumnElement[Any]) -> Select:
"""Return a random sample of up to n rows of the query.

The rows are shuffled before limiting, so the sample isn't biased
towards the order in which the database happens to store them.
"""
return query.order_by(Random()).limit(n)


@compiles(snowflake_types.TIMESTAMP_LTZ)
def compile_timestampltz_generic(
_element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any
Expand Down
21 changes: 11 additions & 10 deletions datafaker/evaluators/column_evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
from sqlalchemy.types import Date, DateTime, Time

from datafaker.db_utils import is_numeric_sql_type
from datafaker.dialects import Random
from datafaker.dialects import random_sample
from datafaker.evaluators.distribution_builders import DEFAULT_SAMPLE_SIZE
from datafaker.evaluators.evaluation_profile import EvaluationProfile
from datafaker.evaluators.metrics import (
DiversityMetric,
Expand Down Expand Up @@ -162,7 +163,7 @@ class ColumnEvaluator: # pylint: disable=too-many-instance-attributes
def __init__(self, columns: list[Column], engine: Engine) -> None:
"""Sample the real column(s) and pick the evaluation profile/criteria."""
assert columns, "ColumnEvaluator needs at least one column"
self.sample_size: int = 4000
self.sample_size: int = DEFAULT_SAMPLE_SIZE
self.engine: Engine = engine
self.novelty_metric: NoveltyMetric = NoveltyMetric()
self.diversity_metric: DiversityMetric = DiversityMetric()
Expand Down Expand Up @@ -194,19 +195,19 @@ def __init__(self, columns: list[Column], engine: Engine) -> None:
# real_uniqueness/copy_fraction/novelty/diversity, all of
# which are computed from this sample.
rows = conn.execute(
select(columns[0])
.select_from(columns[0].table)
.order_by(Random())
.limit(self.sample_size)
random_sample(
select(columns[0]).select_from(columns[0].table),
self.sample_size,
)
)
self.real_values = [row[0] for row in rows]
column_type = get_column_type(columns[0])
else:
rows = conn.execute(
select(*columns)
.select_from(columns[0].table)
.order_by(Random())
.limit(self.sample_size)
random_sample(
select(*columns).select_from(columns[0].table),
self.sample_size,
)
)
self.real_values = [tuple(row) for row in rows]
column_type = None
Expand Down
24 changes: 13 additions & 11 deletions datafaker/evaluators/distribution_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@

from sqlalchemy import func, select

from datafaker.dialects import Random
from datafaker.dialects import random_sample
from datafaker.evaluators.feature_extractors import FeatureExtractor, IdentityExtractor

DEFAULT_SAMPLE_SIZE = 4000


@dataclass
class Distribution:
Expand Down Expand Up @@ -43,7 +45,7 @@ def __init__(
table,
column,
extractor: FeatureExtractor = IdentityExtractor(),
sample_size=4000,
sample_size=DEFAULT_SAMPLE_SIZE,
):
"""Initialize the engine/table/column/extractor/sample_size shared by every builder."""
self.engine = engine
Expand Down Expand Up @@ -73,7 +75,7 @@ def __init__(
column,
bins=10,
extractor: FeatureExtractor = IdentityExtractor(),
sample_size=4000,
sample_size=DEFAULT_SAMPLE_SIZE,
):
"""Initialize a histogram builder for one column feature."""
super().__init__(engine, table, column, extractor, sample_size)
Expand Down Expand Up @@ -104,10 +106,10 @@ def build_from_table(self) -> Distribution:
with self.engine.connect() as conn:
values = (
conn.execute(
select(feature_expr)
.select_from(self.table)
.order_by(Random())
.limit(self.sample_size)
random_sample(
select(feature_expr).select_from(self.table),
self.sample_size,
)
)
.scalars()
.all()
Expand Down Expand Up @@ -227,10 +229,10 @@ def build_from_table(self):
# unordered LIMIT biasing the "real" distribution toward
# whatever the DB returns first (see HistogramBuilder above).
rows = conn.execute(
select(self.column)
.select_from(self.table)
.order_by(Random())
.limit(self.sample_size)
random_sample(
select(self.column).select_from(self.table),
self.sample_size,
)
)

for (value,) in rows:
Expand Down
3 changes: 2 additions & 1 deletion datafaker/evaluators/statistical_fidelity.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from dataclasses import dataclass

from datafaker.evaluators.distribution_builders import (
DEFAULT_SAMPLE_SIZE,
CategoryBuilder,
DistributionBuilder,
HistogramBuilder,
Expand Down Expand Up @@ -223,7 +224,7 @@ class StatisticalFidelity:
in terms of statistical properties.
"""

def __init__(self, column, engine, sample_size=4000):
def __init__(self, column, engine, sample_size=DEFAULT_SAMPLE_SIZE):
"""Initialize with no criteria set; call set_eval_criteria() first."""
self.engine = engine
self.column = column
Expand Down
14 changes: 6 additions & 8 deletions datafaker/interactive/base.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Base configuration command shells."""

import cmd
from abc import ABC, abstractmethod
from collections.abc import Mapping, MutableMapping, Sequence
Expand All @@ -19,7 +20,7 @@
fk_refers_to_ignored_table,
get_sync_engine,
)
from datafaker.dialects import Random
from datafaker.dialects import random_sample
from datafaker.theme import get_active_theme
from datafaker.utils import T, get_property, split_column_full_name

Expand Down Expand Up @@ -469,12 +470,9 @@ def do_peek(self, arg: str) -> None:
return
col_exprs = [table.columns[cn] for cn in col_names]
nonnull_clauses = [ce.isnot(None) for ce in col_exprs]
stmt = (
select(*col_exprs)
.select_from(table)
.where(or_(*nonnull_clauses))
.order_by(Random())
.limit(max_peek_rows)
stmt = random_sample(
select(*col_exprs).select_from(table).where(or_(*nonnull_clauses)),
max_peek_rows,
)
with self.sync_engine.connect() as connection:
try:
Expand Down Expand Up @@ -516,7 +514,7 @@ def get_table_or_column_completions(self, text: str) -> list[str]:
:return: The list of all column names or table names or perhaps
a table name followed by a dot.
"""
(first_part, last_part) = split_column_full_name(text)
first_part, last_part = split_column_full_name(text)
if first_part:
# first_part is table, last_part is column
table_index = self.get_table_index(first_part)
Expand Down
27 changes: 12 additions & 15 deletions datafaker/interactive/generators.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Generator configuration shell.""" # pylint: disable=too-many-lines

import copy
import functools
import re
Expand All @@ -12,7 +13,7 @@
from sqlalchemy.types import Integer

from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private
from datafaker.dialects import Random
from datafaker.dialects import random_sample
from datafaker.evaluators.column_evaluator import ColumnEvaluator
from datafaker.evaluators.evaluation_profile import EvaluationProfile
from datafaker.evaluators.proposal_ranking import (
Expand Down Expand Up @@ -387,7 +388,7 @@ def _get_table_and_proposer(self) -> tuple[str | None, ProposerInfo | None]:

def _get_column_names(self) -> list[str]:
"""Get the (unqualified) names for all the current columns."""
(_, proposer_info) = self._get_table_and_proposer()
_, proposer_info = self._get_table_and_proposer()
return proposer_info.columns if proposer_info else []

def _column_metadata(self) -> list[Column]:
Expand All @@ -399,7 +400,7 @@ def _column_metadata(self) -> list[Column]:

def set_prompt(self) -> None:
"""Set the prompt according to the current table, column and generator."""
(table_name, prop_info) = self._get_table_and_proposer()
table_name, prop_info = self._get_table_and_proposer()
if table_name is None:
self.prompt = "(generators) "
return
Expand Down Expand Up @@ -666,7 +667,7 @@ def _get_table_and_column(self, target: str) -> tuple[int | None, int | None]:
if prop_index is not None:
return (None, prop_index)
# The whole thing isn't a column either, so let's split it:
(first_part, last_part) = split_column_full_name(target)
first_part, last_part = split_column_full_name(target)
if not first_part:
# It doesn't split, so that's the end
self.print(self.ERROR_NO_SUCH_TABLE_OR_COLUMN, last_part)
Expand All @@ -687,7 +688,7 @@ def go_to(self, target: str) -> bool:

:return: True on success.
"""
(table_index, prop_index) = self._get_table_and_column(target)
table_index, prop_index = self._get_table_and_column(target)
if prop_index is None:
return False
if table_index is not None:
Expand Down Expand Up @@ -734,7 +735,7 @@ def complete_next(
self, text: str, _line: str, _begidx: int, _endidx: int
) -> list[str]:
"""Completions for the arguments of the ``next`` command."""
(first_part, last_part) = split_column_full_name(text)
first_part, last_part = split_column_full_name(text)
if first_part:
# first_part is table, last_part is column
table_index = self.get_table_index(first_part)
Expand Down Expand Up @@ -1036,12 +1037,8 @@ def _get_column_data(
table = self.table_metadata()
columns = [table.columns[c] for c in column_names]
nonnull_clauses = [col.isnot(None) for col in columns]
stmt = (
select(*columns)
.select_from(table)
.where(and_(*nonnull_clauses))
.order_by(Random())
.limit(count)
stmt = random_sample(
select(*columns).select_from(table).where(and_(*nonnull_clauses)), count
)
with self.sync_engine.connect() as connection:
result = connection.execute(stmt)
Expand Down Expand Up @@ -1334,7 +1331,7 @@ def do_set(self, arg: str) -> None:

def set_proposer(self, prop: Proposer | None) -> None:
"""Set the current column's generator."""
(table, gen_info) = self._get_table_and_proposer()
table, gen_info = self._get_table_and_proposer()
if table is None:
self.print("Error: no table")
return
Expand Down Expand Up @@ -1578,7 +1575,7 @@ def do_role(self, arg: str) -> None:
parse = self.parse_role(arg)
if parse is None:
return
(column, command) = parse
column, command = parse
if not column:
cols = self.get_current_columns()
if len(cols) != 1:
Expand All @@ -1587,7 +1584,7 @@ def do_role(self, arg: str) -> None:
)
return
column = list(cols)[0]
(table, col) = split_column_full_name(column)
table, col = split_column_full_name(column)
if not table:
entry = self.get_table()
if entry is None:
Expand Down
16 changes: 7 additions & 9 deletions datafaker/interactive/missingness.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
"""Missingness configuration shell."""

import re
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
Expand All @@ -7,7 +8,7 @@
from sqlalchemy import Column, Dialect, Table, func, select
from sqlalchemy.sql.elements import literal_column

from datafaker.dialects import IsNull, Random
from datafaker.dialects import IsNull, random_sample
from datafaker.interactive.base import DbCmd, TableEntry


Expand Down Expand Up @@ -39,13 +40,10 @@ def sampled_query(
:return: The SQL query to do the sampling.
"""
results = [IsNull(c).label(c.name + "__is_null") for c in columns]
subquery = (
select(*results)
.select_from(table)
.order_by(Random())
.limit(literal_column(str(count)))
.subquery("__t")
)
subquery = random_sample(
select(*results).select_from(table),
literal_column(str(count)),
).subquery("__t")
result_labels = [subquery.c[r.name].label(r.name) for r in results]
query = (
select(
Expand Down Expand Up @@ -138,7 +136,7 @@ def make_table_entry(
if isinstance(mg_name, str):
query_comments = self.find_missingness_query(mg)
if query_comments is not None:
(query, comments) = query_comments
query, comments = query_comments
old = MissingnessType(
name=mg_name,
query=query,
Expand Down
7 changes: 4 additions & 3 deletions datafaker/interactive/table.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
"""Table configuration command shell."""

from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any, cast

from sqlalchemy import func, select

from datafaker.dialects import Random
from datafaker.dialects import random_sample
from datafaker.interactive.base import (
TYPE_LETTER,
TYPE_PROMPT,
Expand Down Expand Up @@ -483,7 +484,7 @@ def print_column_data(self, column_name: str, count: int, min_length: int) -> No
where_clause = func.length(column) >= min_length
else:
where_clause = column.isnot(None)
stmt = select(column).where(where_clause).order_by(Random()).limit(count)
stmt = random_sample(select(column).where(where_clause), count)
with self.sync_engine.connect() as connection:
result = connection.execute(stmt)
self.columnize([str(x[0]) for x in result.all()])
Expand All @@ -494,7 +495,7 @@ def print_row_data(self, count: int) -> None:

:param count: The number of rows to report.
"""
stmt = select(self.table_metadata()).order_by(Random()).limit(count)
stmt = random_sample(select(self.table_metadata()), count)
with self.sync_engine.connect() as connection:
result = connection.execute(stmt)
if result is None:
Expand Down
Loading
Loading