Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
157 changes: 96 additions & 61 deletions datafaker/proposers/choice.py
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,9 @@ class ValueGatherer:
from the suppressed values.
"""

def __init__(self, results: CursorResult, suppress_count: int = 0) -> None:
def __init__(
self, results: CursorResult | Sequence[Any], suppress_count: int = 0
) -> None:
"""Initialise a ValueGatherer."""
values = [] # All values found
counts = [] # The number or each value
Expand Down Expand Up @@ -321,6 +323,15 @@ class ChoiceProposerFactory(ProposerFactory):
SAMPLE_COUNT = MAXIMUM_CHOICES
SUPPRESS_COUNT = 7

# If a random sample of SAMPLE_COUNT rows is already at least this
# fraction distinct, skip the exact stmt_count GROUP BY below (see
# get_proposers). By the birthday problem, even if true cardinality
# sat exactly at MAXIMUM_CHOICES, collisions in a same-sized sample
# would be expected to pull its distinct fraction down to ~63%
# (1 - 1/e), not this high - so this is a conservative trigger that
# only fires when cardinality is genuinely far over the cap.
EXACT_SKIP_SAMPLE_DISTINCT_RATIO = 0.9

def get_proposers( # pylint: disable=too-many-locals
self, columns: list[Column], engine: Engine
) -> Sequence[Proposer]:
Expand All @@ -333,64 +344,13 @@ def get_proposers( # pylint: disable=too-many-locals
src_table = column.table # preserves schema for schema-qualified databases
generators = []
with engine.connect() as connection:
stmt_count = (
select(
column.label("v"),
func.count(column).label("f"), # pylint: disable=E1102
)
.select_from(src_table)
.group_by(column)
.order_by(desc(func.count(column))) # pylint: disable=E1102
.limit(MAXIMUM_CHOICES + 1)
)
results = connection.execute(stmt_count)
if results is not None and results.rowcount <= MAXIMUM_CHOICES:
vg = ValueGatherer(results, self.SUPPRESS_COUNT)
if vg.counts:
generators += [
ZipfChoiceProposer(
column,
vg.values,
vg.counts,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values,
vg.counts,
dialect=dialect,
),
WeightedChoiceProposer(
column,
vg.cvs,
vg.counts,
dialect=dialect,
),
]
if vg.counts_not_suppressed:
generators += [
ZipfChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
WeightedChoiceProposer(
column=column,
values=vg.cvs_not_suppressed,
counts=vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
]
# Run the sample first: it's always cheap (bounded by
# SAMPLE_COUNT rows) and, if it comes back almost all-distinct,
# tells us the exact query below is both unaffordable (for an
# unindexed high-cardinality column it can mean a full table
# scan plus a disk-spilling hash aggregation) and pointless
# (its result would be discarded unseen, since rowcount would
# exceed MAXIMUM_CHOICES anyway).
inner = (
select(column.label("v"))
.select_from(src_table)
Expand All @@ -407,8 +367,83 @@ def get_proposers( # pylint: disable=too-many-locals
.order_by(desc(func.count(inner.c.v))) # pylint: disable=E1102
)
sampled_results = connection.execute(stmt_sample)
if sampled_results is not None:
vg = ValueGatherer(sampled_results, self.SUPPRESS_COUNT)
sampled_rows = (
list(sampled_results) if sampled_results is not None else None
)
sample_distinct_count = len(sampled_rows) if sampled_rows is not None else 0
skip_exact = sample_distinct_count >= (
self.SAMPLE_COUNT * self.EXACT_SKIP_SAMPLE_DISTINCT_RATIO
)

if not skip_exact:
stmt_count = (
select(
column.label("v"),
func.count(column).label("f"), # pylint: disable=E1102
)
.select_from(src_table)
.group_by(column)
.limit(MAXIMUM_CHOICES + 1)
)
results = connection.execute(stmt_count)
if results is not None and results.rowcount <= MAXIMUM_CHOICES:
# No ORDER BY above: for a high-cardinality column (the
# common case) this result is discarded unseen below, so
# asking the database to sort it first - potentially
# sorting millions of mostly-tied rows just to throw the
# answer away - is pure waste. When we get here the limit
# never bound (every group was returned), so it's cheap and
# correct to sort in Python instead, preserving the
# most-frequent-first order ZipfChoiceProposer relies on.
rows = sorted(results, key=lambda row: row.f, reverse=True)
vg = ValueGatherer(rows, self.SUPPRESS_COUNT)
if vg.counts:
generators += [
ZipfChoiceProposer(
column,
vg.values,
vg.counts,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values,
vg.counts,
dialect=dialect,
),
WeightedChoiceProposer(
column,
vg.cvs,
vg.counts,
dialect=dialect,
),
]
if vg.counts_not_suppressed:
generators += [
ZipfChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
UniformChoiceProposer(
column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
WeightedChoiceProposer(
column=column,
values=vg.cvs_not_suppressed,
counts=vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
),
]
if sampled_rows is not None:
vg = ValueGatherer(sampled_rows, self.SUPPRESS_COUNT)
if vg.counts:
generators += [
ZipfChoiceProposer(
Expand Down
45 changes: 35 additions & 10 deletions datafaker/providers.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""This module contains Mimesis Provider sub-classes."""
import datetime as dt
import functools
import itertools
import math
import random
from collections.abc import Mapping
Expand Down Expand Up @@ -243,6 +244,17 @@ def zipf_weights(size: int) -> list[float]:
return [1 / (n * total) for n in range(1, size + 1)]


@functools.cache
def zipf_cumulative_weights(size: int) -> list[float]:
"""Get the cumulative Zipf weights for ``random.choices(cum_weights=...)``.

Precomputing the cumulative sum (and caching it by ``size``) avoids
``random.choices`` redoing that O(n) accumulation from ``zipf_weights(size)``
on every single draw.
"""
return list(itertools.accumulate(zipf_weights(size)))


def merge_with_constants(
xs: list[T], constants_at: dict[int, T]
) -> Generator[T, None, None]:
Expand Down Expand Up @@ -288,6 +300,15 @@ def __init__(self, *, seed: int | None = None, **kwargs: Any) -> None:
super().__init__(seed=seed, **kwargs)
np_seed = seed if isinstance(seed, int) else None
self.np_gen = np.random.default_rng(seed=np_seed)
# Keyed by id(a): weighted_choice() is typically called many times
# in a row with the very same list object (once per generated row,
# or once per propose-preview sample), so caching the derived
# (values, cumulative counts) by identity avoids rebuilding them on
# every call. The cache holds a strong reference to `a` itself so
# its id can't be recycled by another list while the entry lives.
self._weighted_choice_cache: dict[
int, tuple[list[dict[str, Any]], list[Any], list[int]]
] = {}

def uniform(self, low: float, high: float) -> float:
"""
Expand Down Expand Up @@ -362,7 +383,7 @@ def zipf_choice_direct(self, a: list[T], n: int | None = None) -> T:
"""
if n is None:
n = len(a)
return random.choices(a, weights=zipf_weights(n))[0]
return random.choices(a, cum_weights=zipf_cumulative_weights(n))[0]

def zipf_choice(self, a: list[Mapping[str, T]], n: int | None = None) -> T | None:
"""
Expand Down Expand Up @@ -390,15 +411,19 @@ def weighted_choice(self, a: list[dict[str, Any]]) -> Any:
"""
if not a:
return None
vs = []
counts = []
for vc in a:
count = vc.get("count", 0)
if count:
counts.append(count)
vs.append(vc.get("value", None))
c = random.choices(vs, weights=counts)[0]
return c
cached = self._weighted_choice_cache.get(id(a))
if cached is None or cached[0] is not a:
vs = []
counts = []
for vc in a:
count = vc.get("count", 0)
if count:
counts.append(count)
vs.append(vc.get("value", None))
cached = (a, vs, list(itertools.accumulate(counts)))
self._weighted_choice_cache[id(a)] = cached
_, vs, cum_counts = cached
return random.choices(vs, cum_weights=cum_counts)[0]

def constant(self, value: T) -> T:
"""Return the same value always."""
Expand Down
56 changes: 32 additions & 24 deletions tests/test_generators_dialect.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,8 +212,16 @@ def test_no_sample_no_suppress_has_no_random_or_limit(self) -> None:
class TestChoiceGeneratorFactoryLiveQueries(DatafakerTestCase):
"""ChoiceGeneratorFactory.get_generators executes dialect-correct live SQL."""

def _captured_sqls(self, dialect, schema=None) -> list[str]:
"""Run get_proposers with a mocked engine and return compiled SQL strings."""
def _captured_sqls(self, dialect, schema=None) -> dict[str, str]:
"""Run get_proposers with a mocked engine; return compiled SQL by query kind.

get_proposers runs the sample query first and the exact stmt_count
query second only if the sample wasn't conclusive (see the
EXACT_SKIP_SAMPLE_DISTINCT_RATIO heuristic in choice.py), so which
one executes first isn't stable across changes to that heuristic.
The two are told apart by content instead: only the sample query
groups on the "_inner" sampling subquery.
"""

engine = MagicMock()
engine.dialect = dialect
Expand All @@ -236,51 +244,51 @@ def _captured_sqls(self, dialect, schema=None) -> list[str]:
conn.__exit__ = MagicMock(return_value=False)
engine.connect.return_value = conn

executed = []
results_queue = [result_count, result_sample]
executed: dict[str, str] = {}

def capture(stmt, *_args, **_kwargs):
executed.append(stmt)
return results_queue[len(executed) - 1]
compiled = str(
stmt.compile(dialect=dialect, compile_kwargs={"literal_binds": True})
).upper()
if "_INNER" in compiled:
executed["sample"] = compiled
return result_sample
executed["exact"] = compiled
return result_count

conn.execute.side_effect = capture

meta = MetaData()
tbl = Table("patient", meta, Column("gender", Integer()), schema=schema)
ChoiceProposerFactory().get_proposers([tbl.c.gender], engine)

return [
str(
s.compile(dialect=dialect, compile_kwargs={"literal_binds": True})
).upper()
for s in executed
]
return executed

def test_mssql_live_queries_use_rand_and_top(self) -> None:
"""MS-SQL live queries use TOP (not LIMIT) and rand() (not random())."""
sqls = self._captured_sqls(mssql.dialect())
self.assert_str_in(" TOP ", sqls[0])
self.assert_str_not_in("LIMIT", sqls[0])
self.assert_str_in("NEWID()", sqls[1])
self.assert_str_not_in("LIMIT", sqls[1])
self.assert_str_not_in("RANDOM()", sqls[1])
self.assert_str_in(" TOP ", sqls["exact"])
self.assert_str_not_in("LIMIT", sqls["exact"])
self.assert_str_in("NEWID()", sqls["sample"])
self.assert_str_not_in("LIMIT", sqls["sample"])
self.assert_str_not_in("RANDOM()", sqls["sample"])

def test_postgresql_live_queries_use_limit_and_random(self) -> None:
"""PostgreSQL live queries use LIMIT and random()."""
sqls = self._captured_sqls(postgresql.dialect())
self.assert_str_in("LIMIT", sqls[0])
self.assert_str_not_in(" TOP ", sqls[0])
self.assert_str_in("LIMIT", sqls[1])
self.assert_str_in("RANDOM()", sqls[1])
self.assert_str_not_in(" TOP ", sqls[1])
self.assert_str_not_in("NEWID()", sqls[1])
self.assert_str_in("LIMIT", sqls["exact"])
self.assert_str_not_in(" TOP ", sqls["exact"])
self.assert_str_in("LIMIT", sqls["sample"])
self.assert_str_in("RANDOM()", sqls["sample"])
self.assert_str_not_in(" TOP ", sqls["sample"])
self.assert_str_not_in("NEWID()", sqls["sample"])

def test_schema_qualified_table_appears_in_from(self) -> None:
"""Schema-qualified table name is included in the FROM clause on both dialects."""
for dialect in (mssql.dialect(), postgresql.dialect()):
with self.subTest(dialect=dialect.name):
sqls = self._captured_sqls(dialect, schema="myschema")
for sql in sqls:
for sql in sqls.values():
self.assert_str_in("MYSCHEMA", sql)


Expand Down
Loading
Loading