diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index 38e475f..784c54c 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -281,7 +281,9 @@ class ValueGatherer: from the suppressed values. """ - def __init__(self, results: CursorResult, suppress_count: int = 0) -> None: + def __init__( + self, results: CursorResult | Sequence[Any], suppress_count: int = 0 + ) -> None: """Initialise a ValueGatherer.""" values = [] # All values found counts = [] # The number or each value @@ -321,6 +323,15 @@ class ChoiceProposerFactory(ProposerFactory): SAMPLE_COUNT = MAXIMUM_CHOICES SUPPRESS_COUNT = 7 + # If a random sample of SAMPLE_COUNT rows is already at least this + # fraction distinct, skip the exact stmt_count GROUP BY below (see + # get_proposers). By the birthday problem, even if true cardinality + # sat exactly at MAXIMUM_CHOICES, collisions in a same-sized sample + # would be expected to pull its distinct fraction down to ~63% + # (1 - 1/e), not this high - so this is a conservative trigger that + # only fires when cardinality is genuinely far over the cap. + EXACT_SKIP_SAMPLE_DISTINCT_RATIO = 0.9 + def get_proposers( # pylint: disable=too-many-locals self, columns: list[Column], engine: Engine ) -> Sequence[Proposer]: @@ -333,64 +344,13 @@ def get_proposers( # pylint: disable=too-many-locals src_table = column.table # preserves schema for schema-qualified databases generators = [] with engine.connect() as connection: - stmt_count = ( - select( - column.label("v"), - func.count(column).label("f"), # pylint: disable=E1102 - ) - .select_from(src_table) - .group_by(column) - .order_by(desc(func.count(column))) # pylint: disable=E1102 - .limit(MAXIMUM_CHOICES + 1) - ) - results = connection.execute(stmt_count) - if results is not None and results.rowcount <= MAXIMUM_CHOICES: - vg = ValueGatherer(results, self.SUPPRESS_COUNT) - if vg.counts: - generators += [ - ZipfChoiceProposer( - column, - vg.values, - vg.counts, - dialect=dialect, - ), - UniformChoiceProposer( - column, - vg.values, - vg.counts, - dialect=dialect, - ), - WeightedChoiceProposer( - column, - vg.cvs, - vg.counts, - dialect=dialect, - ), - ] - if vg.counts_not_suppressed: - generators += [ - ZipfChoiceProposer( - column, - vg.values_not_suppressed, - vg.counts_not_suppressed, - suppress_count=self.SUPPRESS_COUNT, - dialect=dialect, - ), - UniformChoiceProposer( - column, - vg.values_not_suppressed, - vg.counts_not_suppressed, - suppress_count=self.SUPPRESS_COUNT, - dialect=dialect, - ), - WeightedChoiceProposer( - column=column, - values=vg.cvs_not_suppressed, - counts=vg.counts_not_suppressed, - suppress_count=self.SUPPRESS_COUNT, - dialect=dialect, - ), - ] + # Run the sample first: it's always cheap (bounded by + # SAMPLE_COUNT rows) and, if it comes back almost all-distinct, + # tells us the exact query below is both unaffordable (for an + # unindexed high-cardinality column it can mean a full table + # scan plus a disk-spilling hash aggregation) and pointless + # (its result would be discarded unseen, since rowcount would + # exceed MAXIMUM_CHOICES anyway). inner = ( select(column.label("v")) .select_from(src_table) @@ -407,8 +367,83 @@ def get_proposers( # pylint: disable=too-many-locals .order_by(desc(func.count(inner.c.v))) # pylint: disable=E1102 ) sampled_results = connection.execute(stmt_sample) - if sampled_results is not None: - vg = ValueGatherer(sampled_results, self.SUPPRESS_COUNT) + sampled_rows = ( + list(sampled_results) if sampled_results is not None else None + ) + sample_distinct_count = len(sampled_rows) if sampled_rows is not None else 0 + skip_exact = sample_distinct_count >= ( + self.SAMPLE_COUNT * self.EXACT_SKIP_SAMPLE_DISTINCT_RATIO + ) + + if not skip_exact: + stmt_count = ( + select( + column.label("v"), + func.count(column).label("f"), # pylint: disable=E1102 + ) + .select_from(src_table) + .group_by(column) + .limit(MAXIMUM_CHOICES + 1) + ) + results = connection.execute(stmt_count) + if results is not None and results.rowcount <= MAXIMUM_CHOICES: + # No ORDER BY above: for a high-cardinality column (the + # common case) this result is discarded unseen below, so + # asking the database to sort it first - potentially + # sorting millions of mostly-tied rows just to throw the + # answer away - is pure waste. When we get here the limit + # never bound (every group was returned), so it's cheap and + # correct to sort in Python instead, preserving the + # most-frequent-first order ZipfChoiceProposer relies on. + rows = sorted(results, key=lambda row: row.f, reverse=True) + vg = ValueGatherer(rows, self.SUPPRESS_COUNT) + if vg.counts: + generators += [ + ZipfChoiceProposer( + column, + vg.values, + vg.counts, + dialect=dialect, + ), + UniformChoiceProposer( + column, + vg.values, + vg.counts, + dialect=dialect, + ), + WeightedChoiceProposer( + column, + vg.cvs, + vg.counts, + dialect=dialect, + ), + ] + if vg.counts_not_suppressed: + generators += [ + ZipfChoiceProposer( + column, + vg.values_not_suppressed, + vg.counts_not_suppressed, + suppress_count=self.SUPPRESS_COUNT, + dialect=dialect, + ), + UniformChoiceProposer( + column, + vg.values_not_suppressed, + vg.counts_not_suppressed, + suppress_count=self.SUPPRESS_COUNT, + dialect=dialect, + ), + WeightedChoiceProposer( + column=column, + values=vg.cvs_not_suppressed, + counts=vg.counts_not_suppressed, + suppress_count=self.SUPPRESS_COUNT, + dialect=dialect, + ), + ] + if sampled_rows is not None: + vg = ValueGatherer(sampled_rows, self.SUPPRESS_COUNT) if vg.counts: generators += [ ZipfChoiceProposer( diff --git a/datafaker/providers.py b/datafaker/providers.py index 55faabf..5672ebc 100644 --- a/datafaker/providers.py +++ b/datafaker/providers.py @@ -1,6 +1,7 @@ """This module contains Mimesis Provider sub-classes.""" import datetime as dt import functools +import itertools import math import random from collections.abc import Mapping @@ -243,6 +244,17 @@ def zipf_weights(size: int) -> list[float]: return [1 / (n * total) for n in range(1, size + 1)] +@functools.cache +def zipf_cumulative_weights(size: int) -> list[float]: + """Get the cumulative Zipf weights for ``random.choices(cum_weights=...)``. + + Precomputing the cumulative sum (and caching it by ``size``) avoids + ``random.choices`` redoing that O(n) accumulation from ``zipf_weights(size)`` + on every single draw. + """ + return list(itertools.accumulate(zipf_weights(size))) + + def merge_with_constants( xs: list[T], constants_at: dict[int, T] ) -> Generator[T, None, None]: @@ -288,6 +300,15 @@ def __init__(self, *, seed: int | None = None, **kwargs: Any) -> None: super().__init__(seed=seed, **kwargs) np_seed = seed if isinstance(seed, int) else None self.np_gen = np.random.default_rng(seed=np_seed) + # Keyed by id(a): weighted_choice() is typically called many times + # in a row with the very same list object (once per generated row, + # or once per propose-preview sample), so caching the derived + # (values, cumulative counts) by identity avoids rebuilding them on + # every call. The cache holds a strong reference to `a` itself so + # its id can't be recycled by another list while the entry lives. + self._weighted_choice_cache: dict[ + int, tuple[list[dict[str, Any]], list[Any], list[int]] + ] = {} def uniform(self, low: float, high: float) -> float: """ @@ -362,7 +383,7 @@ def zipf_choice_direct(self, a: list[T], n: int | None = None) -> T: """ if n is None: n = len(a) - return random.choices(a, weights=zipf_weights(n))[0] + return random.choices(a, cum_weights=zipf_cumulative_weights(n))[0] def zipf_choice(self, a: list[Mapping[str, T]], n: int | None = None) -> T | None: """ @@ -390,15 +411,19 @@ def weighted_choice(self, a: list[dict[str, Any]]) -> Any: """ if not a: return None - vs = [] - counts = [] - for vc in a: - count = vc.get("count", 0) - if count: - counts.append(count) - vs.append(vc.get("value", None)) - c = random.choices(vs, weights=counts)[0] - return c + cached = self._weighted_choice_cache.get(id(a)) + if cached is None or cached[0] is not a: + vs = [] + counts = [] + for vc in a: + count = vc.get("count", 0) + if count: + counts.append(count) + vs.append(vc.get("value", None)) + cached = (a, vs, list(itertools.accumulate(counts))) + self._weighted_choice_cache[id(a)] = cached + _, vs, cum_counts = cached + return random.choices(vs, cum_weights=cum_counts)[0] def constant(self, value: T) -> T: """Return the same value always.""" diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py index 99261c1..c177620 100644 --- a/tests/test_generators_dialect.py +++ b/tests/test_generators_dialect.py @@ -212,8 +212,16 @@ def test_no_sample_no_suppress_has_no_random_or_limit(self) -> None: class TestChoiceGeneratorFactoryLiveQueries(DatafakerTestCase): """ChoiceGeneratorFactory.get_generators executes dialect-correct live SQL.""" - def _captured_sqls(self, dialect, schema=None) -> list[str]: - """Run get_proposers with a mocked engine and return compiled SQL strings.""" + def _captured_sqls(self, dialect, schema=None) -> dict[str, str]: + """Run get_proposers with a mocked engine; return compiled SQL by query kind. + + get_proposers runs the sample query first and the exact stmt_count + query second only if the sample wasn't conclusive (see the + EXACT_SKIP_SAMPLE_DISTINCT_RATIO heuristic in choice.py), so which + one executes first isn't stable across changes to that heuristic. + The two are told apart by content instead: only the sample query + groups on the "_inner" sampling subquery. + """ engine = MagicMock() engine.dialect = dialect @@ -236,12 +244,17 @@ def _captured_sqls(self, dialect, schema=None) -> list[str]: conn.__exit__ = MagicMock(return_value=False) engine.connect.return_value = conn - executed = [] - results_queue = [result_count, result_sample] + executed: dict[str, str] = {} def capture(stmt, *_args, **_kwargs): - executed.append(stmt) - return results_queue[len(executed) - 1] + compiled = str( + stmt.compile(dialect=dialect, compile_kwargs={"literal_binds": True}) + ).upper() + if "_INNER" in compiled: + executed["sample"] = compiled + return result_sample + executed["exact"] = compiled + return result_count conn.execute.side_effect = capture @@ -249,38 +262,33 @@ def capture(stmt, *_args, **_kwargs): tbl = Table("patient", meta, Column("gender", Integer()), schema=schema) ChoiceProposerFactory().get_proposers([tbl.c.gender], engine) - return [ - str( - s.compile(dialect=dialect, compile_kwargs={"literal_binds": True}) - ).upper() - for s in executed - ] + return executed def test_mssql_live_queries_use_rand_and_top(self) -> None: """MS-SQL live queries use TOP (not LIMIT) and rand() (not random()).""" sqls = self._captured_sqls(mssql.dialect()) - self.assert_str_in(" TOP ", sqls[0]) - self.assert_str_not_in("LIMIT", sqls[0]) - self.assert_str_in("NEWID()", sqls[1]) - self.assert_str_not_in("LIMIT", sqls[1]) - self.assert_str_not_in("RANDOM()", sqls[1]) + self.assert_str_in(" TOP ", sqls["exact"]) + self.assert_str_not_in("LIMIT", sqls["exact"]) + self.assert_str_in("NEWID()", sqls["sample"]) + self.assert_str_not_in("LIMIT", sqls["sample"]) + self.assert_str_not_in("RANDOM()", sqls["sample"]) def test_postgresql_live_queries_use_limit_and_random(self) -> None: """PostgreSQL live queries use LIMIT and random().""" sqls = self._captured_sqls(postgresql.dialect()) - self.assert_str_in("LIMIT", sqls[0]) - self.assert_str_not_in(" TOP ", sqls[0]) - self.assert_str_in("LIMIT", sqls[1]) - self.assert_str_in("RANDOM()", sqls[1]) - self.assert_str_not_in(" TOP ", sqls[1]) - self.assert_str_not_in("NEWID()", sqls[1]) + self.assert_str_in("LIMIT", sqls["exact"]) + self.assert_str_not_in(" TOP ", sqls["exact"]) + self.assert_str_in("LIMIT", sqls["sample"]) + self.assert_str_in("RANDOM()", sqls["sample"]) + self.assert_str_not_in(" TOP ", sqls["sample"]) + self.assert_str_not_in("NEWID()", sqls["sample"]) def test_schema_qualified_table_appears_in_from(self) -> None: """Schema-qualified table name is included in the FROM clause on both dialects.""" for dialect in (mssql.dialect(), postgresql.dialect()): with self.subTest(dialect=dialect.name): sqls = self._captured_sqls(dialect, schema="myschema") - for sql in sqls: + for sql in sqls.values(): self.assert_str_in("MYSCHEMA", sql) diff --git a/tests/test_proposer.py b/tests/test_proposer.py index eda4753..431d738 100644 --- a/tests/test_proposer.py +++ b/tests/test_proposer.py @@ -1,6 +1,7 @@ """Unit tests for proposers.""" import re from pathlib import Path +from typing import Any import duckdb_sqlalchemy import pandas as pd @@ -14,6 +15,8 @@ String, Table, Text, + create_engine, + event, select, text, ) @@ -22,6 +25,7 @@ from datafaker.interactive.generators import get_aggregate_query from datafaker.proposers import ProposerFactory, everything_factory from datafaker.proposers.base import Proposer +from datafaker.proposers.choice import ChoiceProposer, ChoiceProposerFactory from tests.utils import DatafakerTestCase select_re = re.compile( @@ -141,3 +145,96 @@ def try_each_custom_query(self, proposer: Proposer, engine: Engine) -> None: self.fail( f"Custom query from {proposer.name()} {name} failed:\n{query}\n{exc}" ) + + +class ChoiceProposerFactoryOrderingTests(DatafakerTestCase): + """ChoiceProposerFactory.get_proposers must order values by frequency. + + stmt_count's SQL ORDER BY was dropped for performance (sorting it in the + database is pure waste in the common case where the cap check fails and + the whole result is discarded); this guards that the now-Python-side + sort still leaves ZipfChoiceProposer's values most-frequent-first. + """ + + def test_values_are_most_frequent_first(self) -> None: + """Values come back ordered by descending frequency, not insertion order.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("category_table", metadata, Column("cat", Text())) + metadata.create_all(engine) + # Insert in an order that does not match frequency, so a correct + # result can only come from sorting by count, not from physical + # or insertion order. + rows = [{"cat": "rare"}] + [{"cat": "medium"}] * 5 + [{"cat": "common"}] * 10 + with engine.begin() as conn: + conn.execute(table.insert(), rows) + + proposers = ChoiceProposerFactory().get_proposers([table.c.cat], engine) + zipf = next(p for p in proposers if p.function_name() == "dist_gen.zipf_choice") + assert isinstance(zipf, ChoiceProposer) + self.assertEqual(zipf.values, ["common", "medium", "rare"]) + + +class ChoiceProposerFactoryExactSkipTests(DatafakerTestCase): + """The cheap sample can skip the expensive exact stmt_count GROUP BY. + + Prototype for a fix to the (separately-diagnosed) problem where + stmt_count's full-table GROUP BY is catastrophically expensive for an + unindexed, very-high-cardinality column: its result is discarded + unseen anyway whenever true cardinality exceeds MAXIMUM_CHOICES, so if + the cheap SAMPLE_COUNT-row sample we already run is almost entirely + distinct, we can skip stmt_count outright instead of paying for it. + """ + + def _executed_statements(self, engine: Engine, columns: list[Column]) -> list[str]: + """Get the SQL text of every statement executed while getting proposers.""" + statements: list[str] = [] + + # pylint: disable-next=too-many-arguments,too-many-positional-arguments + def _capture(*args: Any, **_kwargs: Any) -> None: + # Signature required by the "before_cursor_execute" event; only + # the SQL text (third positional argument) is needed here. + statements.append(args[2]) + + event.listen(engine, "before_cursor_execute", _capture) + try: + ChoiceProposerFactory().get_proposers(columns, engine) + finally: + event.remove(engine, "before_cursor_execute", _capture) + return statements + + def test_high_cardinality_sample_skips_exact_query(self) -> None: + """Exact stmt_count is skipped when the sample is almost all-distinct.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("wide_table", metadata, Column("val", Text())) + metadata.create_all(engine) + # Every row distinct: a SAMPLE_COUNT-row sample of this table is + # always 100% distinct, well over the skip threshold. + rows = [{"val": f"unique-{i}"} for i in range(2000)] + with engine.begin() as conn: + conn.execute(table.insert(), rows) + + statements = self._executed_statements(engine, [table.c.val]) + # The exact query groups directly on the source table; the sample + # query groups on the "_inner" sampling subquery. Only the latter + # should have run. + self.assertTrue(any("_inner" in s for s in statements)) + self.assertFalse( + any("wide_table" in s and "_inner" not in s for s in statements) + ) + + def test_low_cardinality_sample_still_runs_exact_query(self) -> None: + """Exact stmt_count still runs when cardinality is near/under the cap.""" + engine = create_engine("duckdb:///:memory:") + metadata = MetaData() + table = Table("narrow_table", metadata, Column("val", Text())) + metadata.create_all(engine) + rows = [{"val": f"v{i % 50}"} for i in range(2000)] + with engine.begin() as conn: + conn.execute(table.insert(), rows) + + statements = self._executed_statements(engine, [table.c.val]) + self.assertTrue( + any("narrow_table" in s and "_inner" not in s for s in statements) + ) diff --git a/tests/test_providers.py b/tests/test_providers.py index 2ddd5db..f253a91 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,5 +1,6 @@ """Tests for the providers module.""" import datetime as dt +from collections import Counter from typing import Any from unittest.mock import MagicMock @@ -137,6 +138,51 @@ def test_timespan(self) -> None: self.assertEqual(end - start, delta) +class DistributionProviderChoiceTestCase(DatafakerTestCase): + """Tests for DistributionProvider's weighted_choice/zipf_choice_direct.""" + + def test_weighted_choice_follows_counts(self) -> None: + """weighted_choice should pick values in proportion to their counts.""" + prov = providers.DistributionProvider(seed=0) + a = [ + {"value": "rare", "count": 1}, + {"value": "common", "count": 9}, + {"value": "never", "count": 0}, + ] + counts = Counter(prov.weighted_choice(a) for _ in range(20000)) + self.assertNotIn("never", counts) + observed_odds = counts["common"] / counts["rare"] + self.assertLess(abs(observed_odds / 9 - 1.0), 0.15) + + def test_weighted_choice_empty(self) -> None: + """weighted_choice on an empty list returns None.""" + prov = providers.DistributionProvider(seed=0) + self.assertIsNone(prov.weighted_choice([])) + + def test_weighted_choice_caches_per_list_identity(self) -> None: + """Repeated/alternating calls with distinct lists must not cross-contaminate. + + Regression test for the id(a)-keyed cache added to speed up repeated + calls with the same list object: interleaving two different lists + must not return the wrong (stale) values/weights for either. + """ + prov = providers.DistributionProvider(seed=0) + a = [{"value": "a-only", "count": 1}] + b = [{"value": "b-only", "count": 1}] + seen = {prov.weighted_choice(a) for _ in range(5)} + seen |= {prov.weighted_choice(b) for _ in range(5)} + seen |= {prov.weighted_choice(a) for _ in range(5)} + self.assertEqual(seen, {"a-only", "b-only"}) + + def test_zipf_choice_direct_decreasing_frequency(self) -> None: + """zipf_choice_direct should favour earlier (more frequent) values.""" + prov = providers.DistributionProvider(seed=0) + values = list(range(10)) + counts = Counter(prov.zipf_choice_direct(values) for _ in range(20000)) + frequencies = [counts[v] for v in values] + self.assertEqual(frequencies, sorted(frequencies, reverse=True)) + + class TestWeightedBooleanProvider(DatafakerTestCase): """Tests for WeightedBooleanProvider."""