From 4beeda31424411ec50c9861fa385b0108b209068 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Fri, 6 Nov 2026 21:56:20 +0000 Subject: [PATCH 1/3] refactored random sample --- datafaker/dialects.py | 10 +++++++ datafaker/evaluators/column_evaluator.py | 18 ++++++------- datafaker/evaluators/distribution_builders.py | 18 ++++++------- datafaker/interactive/base.py | 14 +++++----- datafaker/interactive/generators.py | 27 +++++++++---------- datafaker/interactive/missingness.py | 16 +++++------ datafaker/interactive/table.py | 7 ++--- datafaker/proposers/choice.py | 22 +++++---------- datafaker/proposers/continuous.py | 4 +-- datafaker/proposers/extract.py | 7 +++-- datafaker/proposers/intervals.py | 7 ++--- datafaker/providers.py | 5 ++-- 12 files changed, 76 insertions(+), 79 deletions(-) diff --git a/datafaker/dialects.py b/datafaker/dialects.py index 2b0e98c..f01e38d 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -1,4 +1,5 @@ """Dialect differences.""" + import re from collections.abc import Mapping from typing import Any, Optional, TypeVar @@ -518,6 +519,15 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str: return "NEWID()" +def random_sample(query: Select, n: int | ColumnElement[Any]) -> Select: + """Return a random sample of up to n rows of the query. + + The rows are shuffled before limiting, so the sample isn't biased + towards the order in which the database happens to store them. + """ + return query.order_by(Random()).limit(n) + + @compiles(snowflake_types.TIMESTAMP_LTZ) def compile_timestampltz_generic( _element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index feef34e..b1dda34 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -8,7 +8,7 @@ from sqlalchemy.types import Date, DateTime, Time from datafaker.db_utils import is_numeric_sql_type -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.evaluators.evaluation_profile import EvaluationProfile from datafaker.evaluators.metrics import ( DiversityMetric, @@ -194,19 +194,19 @@ def __init__(self, columns: list[Column], engine: Engine) -> None: # real_uniqueness/copy_fraction/novelty/diversity, all of # which are computed from this sample. rows = conn.execute( - select(columns[0]) - .select_from(columns[0].table) - .order_by(Random()) - .limit(self.sample_size) + random_sample( + select(columns[0]).select_from(columns[0].table), + self.sample_size, + ) ) self.real_values = [row[0] for row in rows] column_type = get_column_type(columns[0]) else: rows = conn.execute( - select(*columns) - .select_from(columns[0].table) - .order_by(Random()) - .limit(self.sample_size) + random_sample( + select(*columns).select_from(columns[0].table), + self.sample_size, + ) ) self.real_values = [tuple(row) for row in rows] column_type = None diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index 1278922..4d9b924 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -9,7 +9,7 @@ from sqlalchemy import func, select -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.evaluators.feature_extractors import FeatureExtractor, IdentityExtractor @@ -104,10 +104,10 @@ def build_from_table(self) -> Distribution: with self.engine.connect() as conn: values = ( conn.execute( - select(feature_expr) - .select_from(self.table) - .order_by(Random()) - .limit(self.sample_size) + random_sample( + select(feature_expr).select_from(self.table), + self.sample_size, + ) ) .scalars() .all() @@ -227,10 +227,10 @@ def build_from_table(self): # unordered LIMIT biasing the "real" distribution toward # whatever the DB returns first (see HistogramBuilder above). rows = conn.execute( - select(self.column) - .select_from(self.table) - .order_by(Random()) - .limit(self.sample_size) + random_sample( + select(self.column).select_from(self.table), + self.sample_size, + ) ) for (value,) in rows: diff --git a/datafaker/interactive/base.py b/datafaker/interactive/base.py index ac317dd..bedbfc2 100644 --- a/datafaker/interactive/base.py +++ b/datafaker/interactive/base.py @@ -1,4 +1,5 @@ """Base configuration command shells.""" + import cmd from abc import ABC, abstractmethod from collections.abc import Mapping, MutableMapping, Sequence @@ -19,7 +20,7 @@ fk_refers_to_ignored_table, get_sync_engine, ) -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.theme import get_active_theme from datafaker.utils import T, get_property, split_column_full_name @@ -469,12 +470,9 @@ def do_peek(self, arg: str) -> None: return col_exprs = [table.columns[cn] for cn in col_names] nonnull_clauses = [ce.isnot(None) for ce in col_exprs] - stmt = ( - select(*col_exprs) - .select_from(table) - .where(or_(*nonnull_clauses)) - .order_by(Random()) - .limit(max_peek_rows) + stmt = random_sample( + select(*col_exprs).select_from(table).where(or_(*nonnull_clauses)), + max_peek_rows, ) with self.sync_engine.connect() as connection: try: @@ -516,7 +514,7 @@ def get_table_or_column_completions(self, text: str) -> list[str]: :return: The list of all column names or table names or perhaps a table name followed by a dot. """ - (first_part, last_part) = split_column_full_name(text) + first_part, last_part = split_column_full_name(text) if first_part: # first_part is table, last_part is column table_index = self.get_table_index(first_part) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index cde6d53..c65cc4c 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -1,4 +1,5 @@ """Generator configuration shell.""" # pylint: disable=too-many-lines + import copy import functools import re @@ -12,7 +13,7 @@ from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.evaluators.column_evaluator import ColumnEvaluator from datafaker.evaluators.evaluation_profile import EvaluationProfile from datafaker.evaluators.proposal_ranking import ( @@ -387,7 +388,7 @@ def _get_table_and_proposer(self) -> tuple[str | None, ProposerInfo | None]: def _get_column_names(self) -> list[str]: """Get the (unqualified) names for all the current columns.""" - (_, proposer_info) = self._get_table_and_proposer() + _, proposer_info = self._get_table_and_proposer() return proposer_info.columns if proposer_info else [] def _column_metadata(self) -> list[Column]: @@ -399,7 +400,7 @@ def _column_metadata(self) -> list[Column]: def set_prompt(self) -> None: """Set the prompt according to the current table, column and generator.""" - (table_name, prop_info) = self._get_table_and_proposer() + table_name, prop_info = self._get_table_and_proposer() if table_name is None: self.prompt = "(generators) " return @@ -666,7 +667,7 @@ def _get_table_and_column(self, target: str) -> tuple[int | None, int | None]: if prop_index is not None: return (None, prop_index) # The whole thing isn't a column either, so let's split it: - (first_part, last_part) = split_column_full_name(target) + first_part, last_part = split_column_full_name(target) if not first_part: # It doesn't split, so that's the end self.print(self.ERROR_NO_SUCH_TABLE_OR_COLUMN, last_part) @@ -687,7 +688,7 @@ def go_to(self, target: str) -> bool: :return: True on success. """ - (table_index, prop_index) = self._get_table_and_column(target) + table_index, prop_index = self._get_table_and_column(target) if prop_index is None: return False if table_index is not None: @@ -734,7 +735,7 @@ def complete_next( self, text: str, _line: str, _begidx: int, _endidx: int ) -> list[str]: """Completions for the arguments of the ``next`` command.""" - (first_part, last_part) = split_column_full_name(text) + first_part, last_part = split_column_full_name(text) if first_part: # first_part is table, last_part is column table_index = self.get_table_index(first_part) @@ -1036,12 +1037,8 @@ def _get_column_data( table = self.table_metadata() columns = [table.columns[c] for c in column_names] nonnull_clauses = [col.isnot(None) for col in columns] - stmt = ( - select(*columns) - .select_from(table) - .where(and_(*nonnull_clauses)) - .order_by(Random()) - .limit(count) + stmt = random_sample( + select(*columns).select_from(table).where(and_(*nonnull_clauses)), count ) with self.sync_engine.connect() as connection: result = connection.execute(stmt) @@ -1334,7 +1331,7 @@ def do_set(self, arg: str) -> None: def set_proposer(self, prop: Proposer | None) -> None: """Set the current column's generator.""" - (table, gen_info) = self._get_table_and_proposer() + table, gen_info = self._get_table_and_proposer() if table is None: self.print("Error: no table") return @@ -1578,7 +1575,7 @@ def do_role(self, arg: str) -> None: parse = self.parse_role(arg) if parse is None: return - (column, command) = parse + column, command = parse if not column: cols = self.get_current_columns() if len(cols) != 1: @@ -1587,7 +1584,7 @@ def do_role(self, arg: str) -> None: ) return column = list(cols)[0] - (table, col) = split_column_full_name(column) + table, col = split_column_full_name(column) if not table: entry = self.get_table() if entry is None: diff --git a/datafaker/interactive/missingness.py b/datafaker/interactive/missingness.py index bc41557..4e70701 100644 --- a/datafaker/interactive/missingness.py +++ b/datafaker/interactive/missingness.py @@ -1,4 +1,5 @@ """Missingness configuration shell.""" + import re from collections.abc import Iterable, Mapping from dataclasses import dataclass @@ -7,7 +8,7 @@ from sqlalchemy import Column, Dialect, Table, func, select from sqlalchemy.sql.elements import literal_column -from datafaker.dialects import IsNull, Random +from datafaker.dialects import IsNull, random_sample from datafaker.interactive.base import DbCmd, TableEntry @@ -39,13 +40,10 @@ def sampled_query( :return: The SQL query to do the sampling. """ results = [IsNull(c).label(c.name + "__is_null") for c in columns] - subquery = ( - select(*results) - .select_from(table) - .order_by(Random()) - .limit(literal_column(str(count))) - .subquery("__t") - ) + subquery = random_sample( + select(*results).select_from(table), + literal_column(str(count)), + ).subquery("__t") result_labels = [subquery.c[r.name].label(r.name) for r in results] query = ( select( @@ -138,7 +136,7 @@ def make_table_entry( if isinstance(mg_name, str): query_comments = self.find_missingness_query(mg) if query_comments is not None: - (query, comments) = query_comments + query, comments = query_comments old = MissingnessType( name=mg_name, query=query, diff --git a/datafaker/interactive/table.py b/datafaker/interactive/table.py index 107a53d..f57497b 100644 --- a/datafaker/interactive/table.py +++ b/datafaker/interactive/table.py @@ -1,11 +1,12 @@ """Table configuration command shell.""" + from collections.abc import Mapping from dataclasses import dataclass from typing import Any, cast from sqlalchemy import func, select -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.interactive.base import ( TYPE_LETTER, TYPE_PROMPT, @@ -483,7 +484,7 @@ def print_column_data(self, column_name: str, count: int, min_length: int) -> No where_clause = func.length(column) >= min_length else: where_clause = column.isnot(None) - stmt = select(column).where(where_clause).order_by(Random()).limit(count) + stmt = random_sample(select(column).where(where_clause), count) with self.sync_engine.connect() as connection: result = connection.execute(stmt) self.columnize([str(x[0]) for x in result.all()]) @@ -494,7 +495,7 @@ def print_row_data(self, count: int) -> None: :param count: The number of rows to report. """ - stmt = select(self.table_metadata()).order_by(Random()).limit(count) + stmt = random_sample(select(self.table_metadata()), count) with self.sync_engine.connect() as connection: result = connection.execute(stmt) if result is None: diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index 38e475f..694231a 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -8,7 +8,7 @@ from sqlalchemy import Column, CursorResult, Engine, desc, func, select -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.proposers.base import ( Proposer, ProposerFactory, @@ -58,13 +58,9 @@ def _choice_stmt( # pylint: disable=R0913,R0917 without TOP; this function never emits such a clause. """ if sample_count is not None: - sample_sub = ( - select(column.label("value")) - .where(column.isnot(None)) - .order_by(Random()) - .limit(sample_count) - .subquery("_inner") - ) + sample_sub = random_sample( + select(column.label("value")).where(column.isnot(None)), sample_count + ).subquery("_inner") counted_sub = ( select( sample_sub.c.value, @@ -391,13 +387,9 @@ def get_proposers( # pylint: disable=too-many-locals dialect=dialect, ), ] - inner = ( - select(column.label("v")) - .select_from(src_table) - .order_by(Random()) - .limit(self.SAMPLE_COUNT) - .subquery("_inner") - ) + inner = random_sample( + select(column.label("v")).select_from(src_table), self.SAMPLE_COUNT + ).subquery("_inner") stmt_sample = ( select( inner.c.v, func.count(inner.c.v).label("f") # pylint: disable=E1102 diff --git a/datafaker/proposers/continuous.py b/datafaker/proposers/continuous.py index 33574a3..d72f640 100644 --- a/datafaker/proposers/continuous.py +++ b/datafaker/proposers/continuous.py @@ -26,7 +26,7 @@ from sqlalchemy.types import Integer, Numeric from typing_extensions import Self -from datafaker.dialects import IsNotNull, LogNatural, NullIf, Random, StdDev +from datafaker.dialects import IsNotNull, LogNatural, NullIf, StdDev, random_sample from datafaker.proposers.base import ( Buckets, NumericType, @@ -613,7 +613,7 @@ def _inner_query(self) -> Select: if preds: sel = sel.filter(*preds) if self._sample_count is not None: - sel = sel.order_by(Random()).limit(self._sample_count) + sel = random_sample(sel, self._sample_count) return sel def _middle_query(self, inner_query: Any) -> Any: diff --git a/datafaker/proposers/extract.py b/datafaker/proposers/extract.py index efd960d..f4fe9d3 100644 --- a/datafaker/proposers/extract.py +++ b/datafaker/proposers/extract.py @@ -1,4 +1,5 @@ """Proposers for date member extraction.""" + import datetime from abc import abstractmethod from collections.abc import Mapping, Sequence @@ -7,7 +8,7 @@ from sqlalchemy import Column, Engine, MetaData, select from sqlalchemy.types import Date, DateTime, Integer -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.proposers.base import Proposer, ProposerFactory, get_column_type from datafaker.providers import ExtractProvider @@ -67,9 +68,7 @@ def fit(self, default: float = -1) -> float: def get_based_on_data(self, count: int) -> list[datetime.datetime]: """Get the data from the based-on column in the source data.""" with self._engine.connect() as conn: - rows = conn.execute(select(self._based_on).order_by(Random())).fetchmany( - count - ) + rows = conn.execute(random_sample(select(self._based_on), count)).all() return [row[0] for row in rows] def generate_data(self, count: int) -> list[Any]: diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index 8ff0118..ef17f46 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -1,4 +1,5 @@ """Proposers for date intervals.""" + import datetime from collections.abc import Mapping, Sequence from typing import Any @@ -6,7 +7,7 @@ from sqlalchemy import Column, Engine, MetaData, func, select from sqlalchemy.types import Date, DateTime -from datafaker.dialects import Random, SecondsDifference, StdDev +from datafaker.dialects import SecondsDifference, StdDev, random_sample from datafaker.proposers.base import ( Buckets, ForeignKeyRelationship, @@ -380,7 +381,7 @@ def generate_intervals( """ anchor = self._anchor_related.target query = select(anchor).select_from(anchor.table) - query = query.order_by(Random()).limit(count) + query = random_sample(query, count) with self._engine.connect() as conn: anchors = conn.execute(query).scalars().all() anchors = procrustes( @@ -544,7 +545,7 @@ def generate_data(self, count: int) -> list[Any]: """Generate ``count`` random data points for this column.""" anchor = self._anchor_related.target query = select(anchor).select_from(anchor.table) - query = query.order_by(Random()).limit(count) + query = random_sample(query, count) with self._engine.connect() as conn: return list(conn.execute(query).scalars().all()) diff --git a/datafaker/providers.py b/datafaker/providers.py index 55faabf..b5adaeb 100644 --- a/datafaker/providers.py +++ b/datafaker/providers.py @@ -1,4 +1,5 @@ """This module contains Mimesis Provider sub-classes.""" + import datetime as dt import functools import math @@ -12,7 +13,7 @@ from sqlalchemy import Column, Connection, MetaData from sqlalchemy.sql import func, select -from datafaker.dialects import Random +from datafaker.dialects import random_sample from datafaker.utils import T, logger @@ -29,7 +30,7 @@ def column_value( db_connection: Connection, orm_class: Any, column_name: str ) -> Any: """Return a random value from the column specified.""" - query = select(orm_class).order_by(Random()).limit(1) + query = random_sample(select(orm_class), 1) random_row = db_connection.execute(query).first() if random_row: From 50a6b8194c2a52c83f9029488baa8026af5d15d1 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Thu, 12 Nov 2026 11:07:32 +0000 Subject: [PATCH 2/3] adding test --- tests/test_dialects.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 tests/test_dialects.py diff --git a/tests/test_dialects.py b/tests/test_dialects.py new file mode 100644 index 0000000..d9399e5 --- /dev/null +++ b/tests/test_dialects.py @@ -0,0 +1,54 @@ +"""Tests for the dialect helpers in datafaker.dialects.""" + +import unittest + +from sqlalchemy import column, literal_column, select, table +from sqlalchemy.dialects import mssql, postgresql +from sqlalchemy.engine import Dialect + +from datafaker.dialects import random_sample + + +class TestRandomSample(unittest.TestCase): + """Tests for random_sample.""" + + def _sql(self, query, dialect: Dialect) -> str: + """Compile a query to SQL text for the given dialect.""" + return str( + query.compile(dialect=dialect, compile_kwargs={"literal_binds": True}) + ) + + def _simple(self, n=5): + """A random sample of 'n' rows of column x from table t.""" + return random_sample(select(column("x")).select_from(table("t")), n) + + def test_postgresql_orders_randomly_then_limits(self) -> None: + """PostgreSQL: ORDER BY RANDOM() comes before LIMIT.""" + sql = self._sql(self._simple(), postgresql.dialect()) + self.assertIn("ORDER BY RANDOM()", sql) + self.assertIn("LIMIT 5", sql) + self.assertLess(sql.index("ORDER BY"), sql.index("LIMIT")) + + def test_mssql_uses_top_and_newid(self) -> None: + """MS-SQL: TOP n with ORDER BY NEWID().""" + sql = self._sql(self._simple(), mssql.dialect()) + self.assertIn("TOP 5", sql) + self.assertIn("ORDER BY NEWID()", sql) + + def test_keeps_existing_where_clause(self) -> None: + """A WHERE clause on the query is kept, before the random ordering.""" + query = random_sample( + select(column("x")).select_from(table("t")).where(column("x").isnot(None)), + 5, + ) + sql = self._sql(query, postgresql.dialect()) + self.assertIn("WHERE x IS NOT NULL", sql) + self.assertLess(sql.index("WHERE"), sql.index("ORDER BY")) + + def test_accepts_sql_expression_as_limit(self) -> None: + """The limit can be an SQL expression, as missingness.py passes.""" + query = random_sample( + select(column("x")).select_from(table("t")), literal_column("1000") + ) + sql = self._sql(query, postgresql.dialect()) + self.assertIn("LIMIT 1000", sql) From c430d8049db1b8321714cb95e4fec2d9fa0e58e4 Mon Sep 17 00:00:00 2001 From: nikk-nikaznan Date: Thu, 12 Nov 2026 11:45:38 +0000 Subject: [PATCH 3/3] cleaned up --- datafaker/evaluators/column_evaluator.py | 3 ++- datafaker/evaluators/distribution_builders.py | 6 ++++-- datafaker/evaluators/statistical_fidelity.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/datafaker/evaluators/column_evaluator.py b/datafaker/evaluators/column_evaluator.py index b1dda34..d539462 100644 --- a/datafaker/evaluators/column_evaluator.py +++ b/datafaker/evaluators/column_evaluator.py @@ -9,6 +9,7 @@ from datafaker.db_utils import is_numeric_sql_type from datafaker.dialects import random_sample +from datafaker.evaluators.distribution_builders import DEFAULT_SAMPLE_SIZE from datafaker.evaluators.evaluation_profile import EvaluationProfile from datafaker.evaluators.metrics import ( DiversityMetric, @@ -162,7 +163,7 @@ class ColumnEvaluator: # pylint: disable=too-many-instance-attributes def __init__(self, columns: list[Column], engine: Engine) -> None: """Sample the real column(s) and pick the evaluation profile/criteria.""" assert columns, "ColumnEvaluator needs at least one column" - self.sample_size: int = 4000 + self.sample_size: int = DEFAULT_SAMPLE_SIZE self.engine: Engine = engine self.novelty_metric: NoveltyMetric = NoveltyMetric() self.diversity_metric: DiversityMetric = DiversityMetric() diff --git a/datafaker/evaluators/distribution_builders.py b/datafaker/evaluators/distribution_builders.py index 4d9b924..2ba23ae 100644 --- a/datafaker/evaluators/distribution_builders.py +++ b/datafaker/evaluators/distribution_builders.py @@ -12,6 +12,8 @@ from datafaker.dialects import random_sample from datafaker.evaluators.feature_extractors import FeatureExtractor, IdentityExtractor +DEFAULT_SAMPLE_SIZE = 4000 + @dataclass class Distribution: @@ -43,7 +45,7 @@ def __init__( table, column, extractor: FeatureExtractor = IdentityExtractor(), - sample_size=4000, + sample_size=DEFAULT_SAMPLE_SIZE, ): """Initialize the engine/table/column/extractor/sample_size shared by every builder.""" self.engine = engine @@ -73,7 +75,7 @@ def __init__( column, bins=10, extractor: FeatureExtractor = IdentityExtractor(), - sample_size=4000, + sample_size=DEFAULT_SAMPLE_SIZE, ): """Initialize a histogram builder for one column feature.""" super().__init__(engine, table, column, extractor, sample_size) diff --git a/datafaker/evaluators/statistical_fidelity.py b/datafaker/evaluators/statistical_fidelity.py index 650b52d..5104861 100644 --- a/datafaker/evaluators/statistical_fidelity.py +++ b/datafaker/evaluators/statistical_fidelity.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from datafaker.evaluators.distribution_builders import ( + DEFAULT_SAMPLE_SIZE, CategoryBuilder, DistributionBuilder, HistogramBuilder, @@ -223,7 +224,7 @@ class StatisticalFidelity: in terms of statistical properties. """ - def __init__(self, column, engine, sample_size=4000): + def __init__(self, column, engine, sample_size=DEFAULT_SAMPLE_SIZE): """Initialize with no criteria set; call set_eval_criteria() first.""" self.engine = engine self.column = column