diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py
index cdc7ab3b..b89cc051 100644
--- a/datafaker/db_utils.py
+++ b/datafaker/db_utils.py
@@ -161,23 +161,32 @@ def create_db_engine(
logger.error("DSN %s is malformed: %s", db_dsn, exc)
raise Exit(1) from exc
- settings = {}
+ settings: list[str] = []
+ dialect_name = get_sync_engine(engine).dialect.name
if schema_name is not None:
- if get_sync_engine(engine).dialect.name == "mssql":
- engine = engine.execution_options(schema_translate_map={None: schema_name})
- else:
- settings["search_path"] = schema_name
+ match dialect_name:
+ case "mssql":
+ engine = engine.execution_options(
+ schema_translate_map={None: schema_name}
+ )
+ case "snowflake":
+ settings.append(f"USE SCHEMA {schema_name};")
+ case _:
+ settings.append(f"SET search_path TO {schema_name};")
dataset_views: list[str] = []
if parquet_dir is not None:
- tables = find_parquet_tables(parquet_dir)
- joined = ",".join(
- sorted({str(t.root.parent) for t in tables if not t.is_dataset})
- )
- # double up single quotes
- dj = joined.replace("'", "''")
- # enclose in single quotes
- settings["file_search_path"] = f"'{dj}'"
- dataset_views = [_dataset_view_sql(t) for t in tables if t.is_dataset]
+ match dialect_name:
+ case "duckdb":
+ tables = find_parquet_tables(parquet_dir)
+ # double up single quotes
+ dj = str(parquet_dir).replace("'", "''")
+ settings.append(f"SET file_search_path TO '{dj}';")
+ dataset_views = [_dataset_view_sql(t) for t in tables if t.hive_files()]
+ case _:
+ tables = []
+ logger.error(
+ "--parquet-dir cannot be used with %s, only duckdb.", dialect_name
+ )
if settings or dataset_views:
event_engine = get_sync_engine(engine)
@@ -219,11 +228,15 @@ def create_db_engine_dst(
return create_db_engine(db_dsn, schema_name, use_asyncio)
-def get_metadata(engine: Engine, schema_name: Optional[str] = None) -> MetaData:
+def get_metadata(
+ engine: Engine,
+ schema_name: Optional[str],
+ views_to_tables: bool,
+) -> MetaData:
"""Get the MetaData object associated with the engine passed."""
md = MetaData()
try:
- md.reflect(engine, schema=schema_name)
+ md.reflect(engine, schema=schema_name, views=views_to_tables)
except OperationalError as exc:
logger.error("Cannot connect to database: %s", exc)
raise Exit(1) from exc
@@ -237,7 +250,7 @@ def _sql_quote(text: str, quote: str) -> str:
def _dataset_view_sql(table: ParquetTable) -> str:
"""Make the SQL that exposes a directory of parquet files as one table."""
- files = ",".join(_sql_quote(str(f), "'") for f in table.files)
+ files = ",".join(_sql_quote(str(f), "'") for f in table.hive_files())
return (
f"CREATE OR REPLACE TEMP VIEW {_sql_quote(table.name, chr(34))} AS "
f"SELECT * FROM read_parquet([{files}], "
@@ -259,16 +272,14 @@ def create_dataset_views(
cursor.close()
-def set_db_settings(connection: DBAPIConnection, settings: Mapping[str, str]) -> None:
- """Set the SEARCH_PATH for a PostgreSQL connection."""
+def set_db_settings(connection: DBAPIConnection, settings: list[str]) -> None:
+ """Run commands."""
# https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#remote-schema-table-introspection-and-postgresql-search-path
existing_autocommit = connection.autocommit
connection.autocommit = True
cursor = connection.cursor()
- # Parametrised queries don't work with asyncpg, hence the f-string.
- sql = "".join(f"SET {k} TO {v};" for k, v in settings.items())
- cursor.execute(sql)
+ cursor.execute("\n".join(settings))
cursor.close()
connection.autocommit = existing_autocommit
diff --git a/datafaker/dialects.py b/datafaker/dialects.py
index 9d92e7b3..2b0e98c6 100644
--- a/datafaker/dialects.py
+++ b/datafaker/dialects.py
@@ -3,6 +3,7 @@
from collections.abc import Mapping
from typing import Any, Optional, TypeVar
+import snowflake.sqlalchemy.custom_types as snowflake_types
from sqlalchemy import Column, Select, Table
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.schema import CreateSchema, CreateTable
@@ -515,3 +516,19 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str:
same random number gets chosen for each row.
"""
return "NEWID()"
+
+
+@compiles(snowflake_types.TIMESTAMP_LTZ)
+def compile_timestampltz_generic(
+ _element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any
+) -> str:
+ """Render TIMESTAMP_LTZ as DateTime."""
+ return compiler.process(DateTime(timezone=False), **kw)
+
+
+@compiles(snowflake_types.TIMESTAMP_LTZ, "snowflake")
+def compile_timestampltz_ddb(
+ element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any
+) -> str:
+ """Render TIMESTAMP_LTZ as TIMESTAMP_LTZ."""
+ return compiler.process(element, **kw)
diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py
index ac13e25a..cde6d532 100644
--- a/datafaker/interactive/generators.py
+++ b/datafaker/interactive/generators.py
@@ -8,7 +8,7 @@
from enum import Enum
from typing import Any, Callable, Optional, cast
-from sqlalchemy import Column, and_, literal_column, select
+from sqlalchemy import Column, and_, select
from sqlalchemy.types import Integer
from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private
@@ -1032,12 +1032,13 @@ def _print_select_aggregate_query(self, table_name: str, prop: Proposer) -> None
def _get_column_data(
self, count: int, to_str: Callable[[Any], str] = repr
) -> list[list[str]]:
- columns = self._get_column_names()
- col_exprs = [literal_column(col) for col in columns]
- nonnull_clauses = [literal_column(col).isnot(None) for col in columns]
+ column_names = self._get_column_names()
+ table = self.table_metadata()
+ columns = [table.columns[c] for c in column_names]
+ nonnull_clauses = [col.isnot(None) for col in columns]
stmt = (
- select(*col_exprs)
- .select_from(self.table_metadata())
+ select(*columns)
+ .select_from(table)
.where(and_(*nonnull_clauses))
.order_by(Random())
.limit(count)
diff --git a/datafaker/interactive/missingness.py b/datafaker/interactive/missingness.py
index 60940a36..bc41557c 100644
--- a/datafaker/interactive/missingness.py
+++ b/datafaker/interactive/missingness.py
@@ -215,7 +215,7 @@ def _copy_entries(self) -> None:
"kwargs": {
"patterns": f'SRC_STATS["{src_stat_key}"]["results"]'
},
- "columns": [c.name for c in entry.new_type.columns],
+ "columns": [str(c.name) for c in entry.new_type.columns],
}
]
src_stats.append(
diff --git a/datafaker/interactive/table.py b/datafaker/interactive/table.py
index 7a2852c0..107a53db 100644
--- a/datafaker/interactive/table.py
+++ b/datafaker/interactive/table.py
@@ -3,7 +3,7 @@
from dataclasses import dataclass
from typing import Any, cast
-from sqlalchemy import func, literal_column, select
+from sqlalchemy import func, select
from datafaker.dialects import Random
from datafaker.interactive.base import (
@@ -469,26 +469,21 @@ def complete_data(
return []
return self.get_column_completions(text)
- def print_column_data(self, column: str, count: int, min_length: int) -> None:
+ def print_column_data(self, column_name: str, count: int, min_length: int) -> None:
"""
Print a sample of data from a certain column of the current table.
- :param column: The name of the column to report on.
+ :param column_name: The name of the column to report on.
:param count: The number of rows to sample.
:param min_length: The minimum length of text to choose from (0 for any text).
"""
- col_expr = literal_column(column)
+ table = self.table_metadata()
+ column = table.columns[column_name]
if 0 < min_length:
- where_clause = func.length(col_expr) >= min_length
+ where_clause = func.length(column) >= min_length
else:
- where_clause = col_expr.isnot(None)
- stmt = (
- select(col_expr)
- .select_from(self.table_metadata())
- .where(where_clause)
- .order_by(Random())
- .limit(count)
- )
+ where_clause = column.isnot(None)
+ stmt = select(column).where(where_clause).order_by(Random()).limit(count)
with self.sync_engine.connect() as connection:
result = connection.execute(stmt)
self.columnize([str(x[0]) for x in result.all()])
diff --git a/datafaker/main.py b/datafaker/main.py
index b180b97b..760bb7a5 100644
--- a/datafaker/main.py
+++ b/datafaker/main.py
@@ -172,11 +172,17 @@ def make_tables(
help=(
"Directory of Parquet files to consider part of the database."
" This can be useful when using DuckDB."
+ " The .parquet files or directories containing partitioned tables"
+ " must all be directly inside this named directory."
" Make sure you check the output!"
),
file_okay=False,
dir_okay=True,
),
+ views_to_tables: bool = Option(
+ False,
+ help=("Generate all views as tables in the synthetic version."),
+ ),
) -> None:
"""Make a YAML file representing the tables in the schema.
@@ -192,7 +198,8 @@ def make_tables(
content = make_tables_file(
get_source_dsn(),
get_source_schema(),
- parquet_dir,
+ views_to_tables=views_to_tables,
+ parquet_dir=parquet_dir,
)
orm_file_path.write_text(content, encoding="utf-8")
logger.debug("%s created.", orm_file)
diff --git a/datafaker/make.py b/datafaker/make.py
index d9ddf25b..35fba18d 100644
--- a/datafaker/make.py
+++ b/datafaker/make.py
@@ -10,7 +10,9 @@
from typing import Any, Final, Optional, Tuple, Type
import pandas as pd
+import snowflake.sqlalchemy.custom_types as snowflake_types
import snsql
+import sqlalchemy
import typer
import yaml
from mimesis.providers.base import BaseProvider
@@ -39,7 +41,7 @@
get_sync_engine,
get_vocabulary_table_names,
)
-from datafaker.parquet2orm import get_parquet_orm
+from datafaker.parquet2orm import add_guessed_keys, get_initial_parquet_orm
from datafaker.settings import get_source_dsn, get_source_schema
from datafaker.utils import (
get_columns_assigned,
@@ -58,6 +60,8 @@
if issubclass(entry, BaseProvider) and entry.__module__ == "datafaker.providers":
PROVIDER_IMPORTS.append(entry_name)
+DEFAULT_UNIQUE_CONSTRAINT_TRIES: int | None = 500
+
@dataclass
class VocabularyTableGeneratorInfo:
@@ -378,6 +382,10 @@ def get_result_mappings(
generator=_string_generator,
choice=True,
),
+ snowflake_types.TIMESTAMP_LTZ: GeneratorInfo(
+ generator="generic.datetime.datetime",
+ arg_types={"start": int, "end": int},
+ ),
}
@@ -654,6 +662,8 @@ def get_generation_info(
max_unique_constraint_tries = get_property_or_none(
config, "max-unique-constraint-tries", int
)
+ if max_unique_constraint_tries is None:
+ max_unique_constraint_tries = DEFAULT_UNIQUE_CONSTRAINT_TRIES
return GenerationInfo(
provider_imports=PROVIDER_IMPORTS,
row_generator_module_name=row_generator_module_name,
@@ -694,9 +704,11 @@ def _generate_vocabulary_table(
download_table(table, engine, yaml_file_name, compress)
+# pylint: disable=too-many-arguments disable=too-many-positional-arguments
def make_tables_file(
db_dsn: str,
schema_name: Optional[str],
+ views_to_tables: bool,
parquet_dir: Optional[Path] = None,
engine: Optional[Engine] = None,
) -> str:
@@ -704,16 +716,27 @@ def make_tables_file(
if engine is None:
engine = get_sync_engine(create_db_engine(db_dsn, schema_name=schema_name))
- metadata = get_metadata(engine, schema_name=schema_name)
+ metadata = get_metadata(
+ engine, schema_name=schema_name, views_to_tables=views_to_tables
+ )
meta_dict = metadata_to_dict(metadata, schema_name, engine, parquet_dir)
+ inspector = sqlalchemy.inspect(engine)
+ if hasattr(inspector, "get_view_names"):
+ view_names = set(inspector.get_view_names())
+ else:
+ view_names = set()
+ parquet_table_names = set()
if parquet_dir is not None:
- extra_meta = get_parquet_orm(parquet_dir)
+ extra_meta = get_initial_parquet_orm(parquet_dir)
if extra_meta:
+ parquet_table_names = set(extra_meta.keys())
md_tables: dict[str, Any] = get_property(meta_dict, "tables", {})
new_tables = {**extra_meta, **md_tables}
meta_dict["tables"] = new_tables
+ add_guessed_keys(meta_dict["tables"], view_names | parquet_table_names)
+
return yaml.dump(meta_dict, sort_keys=False)
diff --git a/datafaker/parquet2orm.py b/datafaker/parquet2orm.py
index a8d5030e..e424f7ed 100644
--- a/datafaker/parquet2orm.py
+++ b/datafaker/parquet2orm.py
@@ -1,15 +1,15 @@
"""Add to ORM structure based on parquet files."""
+import glob
import re
-from collections.abc import Iterable, Mapping
-from dataclasses import dataclass
+from abc import ABC
+from collections.abc import Generator, Mapping
from pathlib import Path
from typing import Any
import numpy as np
-from fastparquet import ParquetFile
-from typer import Exit
+from fastparquet import ParquetException, ParquetFile
-from datafaker.utils import logger
+from datafaker.utils import get_property, logger
PARQUET_SUFFIXES = {".parquet", ".parq"}
@@ -17,35 +17,36 @@
# Partition keys are not stored in the files themselves.
PARTITION_KEY_DTYPE = "category"
-_hive_dir_re = re.compile(r"^([^=]+)=(.*)$")
+KEY_TYPES = frozenset(["INTEGER", "UUID"])
-@dataclass(frozen=True)
-class ParquetTable:
- """A table found in a directory of parquet files."""
+class ParquetTable(ABC):
+ """A table found in a parquet file or directory of parquet files."""
- name: str
- """The name of the table, as it will appear in ``orm.yaml``."""
- root: Path
- """The parquet file for a single-file table, or the directory of a dataset."""
- files: tuple[Path, ...]
- """All the parquet files that make up the table, sorted."""
- is_dataset: bool
- """True if the table is a (possibly partitioned) directory of parquet files."""
+ def __init__(self, pf: ParquetFile, name: str) -> None:
+ """Initialize a ParquetTable."""
+ self.parquet_file = pf
+ self.name = name
+ def hive_files(self) -> list[Path]:
+ """Get any files needing to be merged into a view."""
+ return []
-def _parquet_files_under(directory: Path) -> tuple[Path, ...]:
- """Find all the parquet files in ``directory`` and below, sorted."""
- return tuple(
- sorted(
- entry
- for entry in directory.rglob("*")
- if entry.suffix in PARQUET_SUFFIXES and entry.is_file()
- )
- )
+class ParquetHiveTable(ParquetTable):
+ """A hive partitioned table formed from a directory."""
-def find_parquet_tables(directory: Path | str) -> list[ParquetTable]:
+ def __init__(self, pf: ParquetFile, name: str, file_list: list[Path]):
+ """Initialize a ParquetHiveTable."""
+ super().__init__(pf, name)
+ self.file_list = file_list
+
+ def hive_files(self) -> list[Path]:
+ """Return the list of files that make up this hive."""
+ return self.file_list
+
+
+def find_parquet_tables(directory: Path | str) -> Generator[ParquetTable, None, None]:
"""
Find the tables represented by the parquet files in a directory.
@@ -57,129 +58,72 @@ def find_parquet_tables(directory: Path | str) -> list[ParquetTable]:
(named ``key=value``), then ``directory`` is itself a single table.
:param directory: The directory to search for parquet files in.
- :return: The tables found, files first, then datasets, each sorted by name.
+ :return: The ParquetFiles found, sorted by name.
"""
directory = Path(directory)
- children = sorted(directory.iterdir())
- file_tables = [
- ParquetTable(entry.name, entry, (entry,), False)
- for entry in children
- if entry.suffix in PARQUET_SUFFIXES and entry.is_file()
- ]
- dir_files = [
- (entry, files)
- for entry in children
- if entry.is_dir()
- for files in [_parquet_files_under(entry)]
- if files
- ]
- if not file_tables and dir_files:
- if all(_hive_dir_re.match(entry.name) for entry, _ in dir_files):
- name = directory.resolve().name
- return [
- ParquetTable(
- name,
- directory,
- tuple(f for _, files in dir_files for f in files),
- True,
- )
- ]
- return file_tables + [
- ParquetTable(entry.name, entry, files, True) for entry, files in dir_files
- ]
-
-
-def _file_dtypes(path: Path) -> dict[str, Any]:
- """Get the column names and dtypes stored in one parquet file."""
- return dict(ParquetFile(path).dtypes)
-
-
-def _partition_keys(table: ParquetTable, path: Path) -> list[str]:
- """Get the Hive partition keys of one file of a dataset, in path order."""
- keys = []
- for part in path.relative_to(table.root).parts[:-1]:
- match = _hive_dir_re.match(part)
- if match:
- keys.append(match.group(1))
- return keys
-
-
-def _describe_column_difference(
- expected: Mapping[str, Any], actual: Mapping[str, Any]
-) -> str:
- """Describe how the column names of ``actual`` differ from ``expected``."""
- problems = []
- missing = [c for c in expected if c not in actual]
- extra = [c for c in actual if c not in expected]
- if missing:
- problems.append(f"missing columns {missing}")
- if extra:
- problems.append(f"extra columns {extra}")
- return "; ".join(problems)
-
-
-def _type_name(dtype: Any) -> str:
- """Get a name for a dtype that ignores case, so ``int32`` equals ``Int32``."""
- return str(dtype).lower()
-
-
-def get_table_dtypes(table: ParquetTable) -> dict[str, Any]:
+ for child in sorted(directory.iterdir()):
+ try:
+ if child.is_dir():
+ files = glob.glob("**/*.parquet", root_dir=child, recursive=True)
+ abs_files = [child / f for f in files]
+ pf = ParquetFile(abs_files, root=child, verify=True)
+ yield ParquetHiveTable(pf, child.name, abs_files)
+ else:
+ pf = ParquetFile(child, verify=True)
+ yield ParquetTable(pf, child.name)
+ except ParquetException as e:
+ logger.error("Entry %s cannot be read as parquet", child.name)
+ logger.warning(e)
+ except ValueError as e:
+ logger.error("Entry %s cannot be read as parquet", child.name)
+ logger.warning(e)
+ except IndexError as _e:
+ logger.warning("Entry %s is empty", child.name)
+
+
+def add_guessed_keys(tables_conf: dict[str, Any], table_names: set[str]):
"""
- Get the columns and dtypes of a table, checking that its files agree.
-
- All the files of a partitioned table must have the same column names and
- the same partition keys. Directories that contain unrelated tables are
- rejected with an error message. Column types may differ between files
- (for example when a partition has no values for a column); this is
- reported as a warning and the type from the first file is used.
+ Read the orm.yaml config "tables" portion, adding guesses for foreign keys and primary keys.
- :param table: The table to examine.
- :return: A dict of column names to dtypes. Partition keys are included.
- :raises typer.Exit: If the files of a dataset do not all have the same columns.
+ :param tables_conf: On input, the tables portion of the configuration file; this dictionary will
+ be changed to set the appropriate primary and foreign keys.
+ :param table_names: The names of the tables to guess keys for.
"""
- first = table.files[0]
- dtypes = _file_dtypes(first)
- if not table.is_dataset:
- return dtypes
- keys = _partition_keys(table, first)
- type_names: dict[str, set[str]] = {}
- for path in table.files[1:]:
- file_dtypes = _file_dtypes(path)
- problem = _describe_column_difference(dtypes, file_dtypes)
- if not problem and _partition_keys(table, path) != keys:
- problem = "different partition directories"
- if problem:
- logger.error(
- "Directory %s cannot be read as a single partitioned table: "
- "%s differs from %s (%s). All the files in a partitioned "
- "table must have the same columns. If the "
- "directories inside %s hold different tables, move each "
- "table into its own directory directly inside the parquet "
- "directory and run this command again.",
- table.name,
- path,
- first,
- problem,
- table.root,
+ guesser = _ColumnGuesser(tables_conf)
+ for table_name in table_names:
+ table_conf = tables_conf.get(table_name, {})
+ tables_conf[table_name] = table_conf
+ columns = get_property(table_conf, "columns", {})
+ table_conf["columns"] = columns
+ primaries = set()
+ for column_name, column_conf in columns.items():
+ ctype = get_property(column_conf, "type", "INTEGER")
+ if ctype in KEY_TYPES:
+ if guesser.is_likely_primary(table_name, column_name):
+ primaries.add(column_name)
+ column_conf["primary"] = True
+ column_conf["nullable"] = False
+ elif (
+ guess := guesser.get_likely_foreign_key_target(column_name)
+ ) and guess[0] != table_name:
+ column_conf["foreign_keys"] = [f"{guess[0]}.{guess[1]}"]
+ if column_name == "id":
+ column_conf["primary"] = True
+ column_conf["nullable"] = False
+ if len(primaries) == 0:
+ logger.warning(
+ "Found no likely primary keys for table %s",
+ table_name,
+ )
+ elif 1 < len(primaries):
+ logger.warning(
+ "Found multiple likely primary keys for table %s: %s",
+ table_name,
+ ", ".join(sorted(list(primaries))),
)
- raise Exit(1)
- for column, dtype in file_dtypes.items():
- if _type_name(dtype) != _type_name(dtypes[column]):
- names = type_names.setdefault(column, {_type_name(dtypes[column])})
- names.add(_type_name(dtype))
- for column, names in type_names.items():
- logger.warning(
- "Column %s.%s has different types in different files: %s. "
- "Using the type from %s.",
- table.name,
- column,
- sorted(names),
- first,
- )
- return {**dtypes, **{k: PARTITION_KEY_DTYPE for k in keys if k not in dtypes}}
-
-
-def get_parquet_orm(directory: Path) -> dict[str, Any] | None:
+
+
+def get_initial_parquet_orm(directory: Path) -> dict[str, Any] | None:
"""
Read the parquet files, guess a database structure from that.
@@ -193,13 +137,8 @@ def get_parquet_orm(directory: Path) -> dict[str, Any] | None:
if not directory.is_dir():
logger.error("%s is not a directory", directory)
return None
- tables = find_parquet_tables(directory)
- dtypes = {table.name: get_table_dtypes(table) for table in tables}
- guesser = _ColumnGuesser(dtypes)
- return {
- name: _get_table_orm(table_dtypes, name, guesser)
- for name, table_dtypes in dtypes.items()
- }
+ parquets = find_parquet_tables(directory)
+ return {p.name: _get_table_orm(p.parquet_file) for p in parquets}
_camel_case_re = re.compile(r"([a-z])([A-Z])")
@@ -215,14 +154,21 @@ def _get_words(s: str) -> set[str]:
class _ColumnGuesser:
"""Guesses foreign key targets."""
- def __init__(self, tables: Mapping[str, Iterable[str]]) -> None:
- """Initialize the column guesser from table names to column names."""
+ def __init__(self, tables: Mapping[str, Any]) -> None:
+ """
+ Initialize the column guesser from table names to column names.
+
+ :param tables: The "tables" key from the orm.yaml structure.
+ """
+ # List of triples (table_name, column_name, list_of_words_in_table_and_column_name)
self.tn_cn_words: list[tuple[str, str, set[str]]] = []
+ # Dict from table name to the words in that table name
self.tn2words: dict[str, set[str]] = {}
- for name, table in tables.items():
+ for name, table_conf in tables.items():
table_words = _get_words(name)
self.tn2words[name] = table_words
- for column in table:
+ columns = get_property(table_conf, "columns", {})
+ for column in columns.keys():
self.tn_cn_words.append(
(name, column, table_words | _get_words(column))
)
@@ -235,7 +181,8 @@ def get_likely_foreign_key_target(
Get a foreign key target string if the name suggests it might be.
:param column_name: The name of the column which might be a foreign key
- :return: "table.column" or None if no column seems like a likely target
+ :return: A pair (table_name, column_name) or None if no column seems
+ like a likely target.
"""
our_words = _get_words(column_name)
max_overlaps_so_far = (1, len(our_words) - 1, 0)
@@ -256,64 +203,25 @@ def get_likely_foreign_key_target(
best_so_far = (tn, cn)
return best_so_far
+ def is_likely_primary(self, table_name, column_name) -> bool:
+ """Determine if the specified column name looks like a primary key."""
+ words = _get_words(column_name) - {"id", "key"}
+ return words <= self.tn2words[table_name]
-def _get_table_orm(
- dtypes: Mapping[str, Any], name: str, column_guesser: _ColumnGuesser
-) -> dict[str, Any]:
+
+def _get_table_orm(parquet_file: ParquetFile) -> dict[str, Any]:
"""
Guess the ORM configuration of the table passed in.
- :param dtypes: The column names and dtypes of the table to guess the config of.
- :param name: The filename of the parquet file, or directory name of the dataset.
- :param column_guesser: Guesses foreign key targets across all the tables.
+ :param dtypes: The parquet file to make the table configuration for.
:return: A reasonable ORM configuration for this table.
"""
- column_types = {column: _dtype_to_sql(dtype) for column, dtype in dtypes.items()}
- # Get everything before the last dot
- # (or the whole thing if there is no dot)
- name_pref = name.rsplit(".", 1)[0]
- name_words = _get_words(name_pref)
- cols_orm = {}
- likely_primaries = []
- for column, ctype in column_types.items():
- # A primary key is likely to be the column name plus "_id" or something
- words = _get_words(column) - {"id", "key"}
- col_orm: dict[str, Any] = {}
- if ctype == "INTEGER":
- if words <= name_words:
- likely_primaries.append(column)
- col_orm["primary"] = True
- col_orm["nullable"] = False
- else:
- col_orm["nullable"] = True
- fk_pair = column_guesser.get_likely_foreign_key_target(column)
- if fk_pair is not None:
- fk_table, fk_column = fk_pair
- if fk_table != name:
- logger.debug(
- "Column %s.%s guessed as being a foreign key to %s.%s",
- name_pref,
- column,
- fk_table,
- fk_column,
- )
- col_orm["foreign_keys"] = [f"{fk_table}.{fk_column}"]
- if ctype is not None:
- logger.debug("Column %s.%s type guessed as %s", name_pref, column, ctype)
- col_orm["type"] = ctype
- else:
- logger.warning(
- "Could not determine type of column %s.%s", name_pref, column
- )
- cols_orm[column] = col_orm
- if len(likely_primaries) == 0:
- logger.warning("No likely primary keys found for table %s", name)
- elif 1 < len(likely_primaries):
- logger.warning(
- "Found multiple likely primary keys for table %s: %s",
- name,
- likely_primaries,
- )
+ cols_orm = {
+ column: {
+ "type": _dtype_to_sql(dtype),
+ }
+ for column, dtype in parquet_file.dtypes.items()
+ }
return {
"columns": cols_orm,
"unique": [],
diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py
index acf0e1f4..38e475f2 100644
--- a/datafaker/proposers/choice.py
+++ b/datafaker/proposers/choice.py
@@ -6,17 +6,7 @@
from abc import abstractmethod
from typing import Any, Sequence, Union
-from sqlalchemy import (
- Column,
- CursorResult,
- Engine,
- desc,
- func,
- literal_column,
- select,
- table,
- text,
-)
+from sqlalchemy import Column, CursorResult, Engine, desc, func, select
from datafaker.dialects import Random
from datafaker.proposers.base import (
@@ -25,7 +15,6 @@
dist_gen,
fit_from_buckets,
)
-from datafaker.utils import schema_qualified_name
NumericType = Union[int, float]
@@ -57,12 +46,10 @@ def zipf_distribution(total: int, bins: int) -> typing.Generator[int, None, None
def _choice_stmt( # pylint: disable=R0913,R0917
- column_name: str,
- table_name: str,
+ column: Column,
store_counts: bool,
sample_count: int | None,
suppress_count: int,
- table_sql: str | None = None,
) -> Any:
"""Build a SQLAlchemy SELECT for gathering choice value distributions.
@@ -70,13 +57,10 @@ def _choice_stmt( # pylint: disable=R0913,R0917
TOP/newid() on MS-SQL. MS-SQL also forbids ORDER BY inside a subquery
without TOP; this function never emits such a clause.
"""
- col = literal_column(f'"{column_name}"')
- tbl = text(table_sql) if table_sql else table(table_name)
if sample_count is not None:
sample_sub = (
- select(col.label("value"))
- .where(col.isnot(None))
- .select_from(tbl)
+ select(column.label("value"))
+ .where(column.isnot(None))
.order_by(Random())
.limit(sample_count)
.subquery("_inner")
@@ -92,12 +76,11 @@ def _choice_stmt( # pylint: disable=R0913,R0917
else:
counted_sub = (
select(
- col.label("value"),
- func.count(col).label("count"), # pylint: disable=E1102
+ column.label("value"),
+ func.count(column).label("count"), # pylint: disable=E1102
)
- .where(col.isnot(None))
- .select_from(tbl)
- .group_by(col)
+ .where(column.isnot(None))
+ .group_by(column)
.subquery("_counted")
)
out_cols = [counted_sub.c.value]
@@ -119,31 +102,26 @@ class ChoiceProposer(Proposer):
# pylint: disable=too-many-arguments too-many-positional-arguments
def __init__(
self,
- table_name: str,
- column_name: str,
+ column: Column,
values: list[Any],
counts: list[int],
sample_count: int | None = None,
suppress_count: int = 0,
dialect: Any = None,
- table_sql: str | None = None,
) -> None:
"""Initialise a ChoiceProposer."""
super().__init__()
- self.table_name = table_name
- self.column_name = column_name
+ self.column = column
self.values = values
estimated_counts = self.get_estimated_counts(counts)
self._fit = fit_from_buckets(counts, estimated_counts)
extra_comment = " and their counts" if self.STORE_COUNTS else ""
stmt = _choice_stmt(
- column_name,
- table_name,
+ column,
self.STORE_COUNTS,
sample_count,
suppress_count,
- table_sql=table_sql,
)
compile_opts: dict[str, Any] = {"compile_kwargs": {"literal_binds": True}}
if dialect is not None:
@@ -153,28 +131,28 @@ def __init__(
if suppress_count == 0:
if sample_count is None:
self._comment = (
- f"All the values{extra_comment} that appear in column {column_name}"
- f" of table {table_name}"
+ f"All the values{extra_comment} that appear in column {column.name}"
+ f" of table {column.table.name}"
)
self._annotation = None
else:
self._comment = (
- f"The values{extra_comment} that appear in column {column_name}"
- f" of a random sample of {sample_count} rows of table {table_name}"
+ f"The values{extra_comment} that appear in column {column.name}"
+ f" of a random sample of {sample_count} rows of table {column.table.name}"
)
self._annotation = "sampled"
else:
if sample_count is None:
self._comment = (
- f"All the values{extra_comment} that appear in column {column_name}"
- f" of table {table_name} more than {suppress_count} times"
+ f"All the values{extra_comment} that appear in column {column.name}"
+ f" of table {column.table.name} more than {suppress_count} times"
)
self._annotation = "suppressed"
else:
self._comment = (
f"The values{extra_comment} that appear more than {suppress_count} times"
- f" in column {column_name}, out of a random sample of {sample_count} rows"
- f" of table {table_name}"
+ f" in column {column.name}, out of a random sample of {sample_count} rows"
+ f" of table {column.table.name}"
)
self._annotation = "sampled and suppressed"
@@ -185,7 +163,7 @@ def get_estimated_counts(self, counts: list[int]) -> list[int]:
def nominal_kwargs(self) -> dict[str, Any]:
"""Get the arguments to be entered into ``config.yaml``."""
return {
- "a": f'SRC_STATS["auto__{self.table_name}__{self.column_name}"]["results"]',
+ "a": f'SRC_STATS["auto__{self.column.table.name}__{self.column.name}"]["results"]',
}
def name(self) -> str:
@@ -206,7 +184,7 @@ def custom_queries(self) -> dict[str, dict[str, Any]]:
qs = super().custom_queries()
return {
**qs,
- f"auto__{self.table_name}__{self.column_name}": {
+ f"auto__{self.column.table.name}__{self.column.name}": {
"query": self._query,
"comments": [self._comment],
},
@@ -350,23 +328,19 @@ def get_proposers( # pylint: disable=too-many-locals
if len(columns) != 1:
return []
column = columns[0]
- column_name = column.name
- table_name = column.table.name
- table_sql = schema_qualified_name(table_name, engine)
src_table = column.table
dialect = engine.dialect
- col = literal_column(f'"{column_name}"')
src_table = column.table # preserves schema for schema-qualified databases
generators = []
with engine.connect() as connection:
stmt_count = (
select(
- col.label("v"),
- func.count(col).label("f"), # pylint: disable=E1102
+ column.label("v"),
+ func.count(column).label("f"), # pylint: disable=E1102
)
.select_from(src_table)
- .group_by(col)
- .order_by(desc(func.count(col))) # pylint: disable=E1102
+ .group_by(column)
+ .order_by(desc(func.count(column))) # pylint: disable=E1102
.limit(MAXIMUM_CHOICES + 1)
)
results = connection.execute(stmt_count)
@@ -375,62 +349,50 @@ def get_proposers( # pylint: disable=too-many-locals
if vg.counts:
generators += [
ZipfChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values,
vg.counts,
dialect=dialect,
- table_sql=table_sql,
),
UniformChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values,
vg.counts,
dialect=dialect,
- table_sql=table_sql,
),
WeightedChoiceProposer(
- table_name,
- column_name,
+ column,
vg.cvs,
vg.counts,
dialect=dialect,
- table_sql=table_sql,
),
]
if vg.counts_not_suppressed:
generators += [
ZipfChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
UniformChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
WeightedChoiceProposer(
- table_name=table_name,
- column_name=column_name,
+ column=column,
values=vg.cvs_not_suppressed,
counts=vg.counts_not_suppressed,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
]
inner = (
- select(col.label("v"))
+ select(column.label("v"))
.select_from(src_table)
.order_by(Random())
.limit(self.SAMPLE_COUNT)
@@ -450,64 +412,52 @@ def get_proposers( # pylint: disable=too-many-locals
if vg.counts:
generators += [
ZipfChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values,
vg.counts,
sample_count=self.SAMPLE_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
UniformChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values,
vg.counts,
sample_count=self.SAMPLE_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
WeightedChoiceProposer(
- table_name,
- column_name,
+ column,
vg.cvs,
vg.counts,
sample_count=self.SAMPLE_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
]
if vg.counts_not_suppressed:
generators += [
ZipfChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
sample_count=self.SAMPLE_COUNT,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
UniformChoiceProposer(
- table_name,
- column_name,
+ column,
vg.values_not_suppressed,
vg.counts_not_suppressed,
sample_count=self.SAMPLE_COUNT,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
WeightedChoiceProposer(
- table_name=table_name,
- column_name=column_name,
+ column=column,
values=vg.cvs_not_suppressed,
counts=vg.counts_not_suppressed,
sample_count=self.SAMPLE_COUNT,
suppress_count=self.SUPPRESS_COUNT,
dialect=dialect,
- table_sql=table_sql,
),
]
return generators
diff --git a/datafaker/proposers/mimesis.py b/datafaker/proposers/mimesis.py
index 451ad792..dbb9282f 100644
--- a/datafaker/proposers/mimesis.py
+++ b/datafaker/proposers/mimesis.py
@@ -5,7 +5,7 @@
import mimesis
import mimesis.locales
-from sqlalchemy import Column, Engine, cast, extract, func, literal_column, select
+from sqlalchemy import Column, Engine, cast, extract, func, select
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.types import Date, DateTime, Integer, Numeric, String, Time
@@ -187,8 +187,7 @@ def make_singleton(
cls, column: Column, engine: Engine, function_name: str
) -> Sequence[Proposer]:
"""Make the appropriate generation configuration for this column."""
- col_expr = literal_column(column.name)
- year_expr = cast(extract("year", col_expr), Integer())
+ year_expr = cast(extract("year", column), Integer())
min_expr = func.min(year_expr) # pylint: disable=E1111
max_expr = func.max(year_expr) # pylint: disable=E1111
stmt = select(min_expr.label("start"), max_expr.label("end")).select_from(
diff --git a/datafaker/remove.py b/datafaker/remove.py
index 0949e048..a6eae71c 100644
--- a/datafaker/remove.py
+++ b/datafaker/remove.py
@@ -68,5 +68,9 @@ def remove_db_tables(metadata: Optional[MetaData]) -> None:
)
)
if metadata is None:
- metadata = get_metadata(dst_engine, schema_name=schema_name)
+ metadata = get_metadata(
+ dst_engine,
+ schema_name=schema_name,
+ views_to_tables=False,
+ )
metadata.drop_all(dst_engine)
diff --git a/datafaker/serialize_metadata.py b/datafaker/serialize_metadata.py
index 35a066b7..fc2b555c 100644
--- a/datafaker/serialize_metadata.py
+++ b/datafaker/serialize_metadata.py
@@ -6,6 +6,7 @@
from typing import Any
import parsy
+import snowflake.sqlalchemy.custom_types as snowflake_types
from sqlalchemy import Column, Dialect, Engine, ForeignKey, MetaData, Table
from sqlalchemy.dialects import mssql, oracle, postgresql
from sqlalchemy.sql import schema, sqltypes
@@ -103,7 +104,7 @@ def time_type(type_: type, tz_type: type) -> ParserType:
Make a parser for a SQL date/time type.
Parses TYPE_NAME, TYPE_NAME(32), TYPE_NAME WITH TIME ZONE
- or TYPE_NAME(32) WITH TIME ZONE
+ or TYPE_NAME(32) WITH TIME ZONE, also WITHOUT
:param type_: The SQLAlchemy type we would like to parse.
:param tz_type: The type to instantiate when precision or timezone is
@@ -204,6 +205,15 @@ def _mssql_varbinary_parser() -> typing.Generator[ParserType, None, typing.Any]:
string_type(sqltypes.VARCHAR),
string_type(sqltypes.NVARCHAR),
string_type(sqltypes.TEXT),
+ parsy.string("TIMESTAMP_NTZ").result(sqltypes.DATETIME),
+ parsy.string("TIMESTAMPNTZ").result(sqltypes.DATETIME),
+ parsy.string("TIMESTAMP_LTZ").result(snowflake_types.TIMESTAMP_LTZ),
+ parsy.string("TIMESTAMPLTZ").result(snowflake_types.TIMESTAMP_LTZ),
+ parsy.string("TIMESTAMP WITH LOCAL TIME ZONE").result(
+ snowflake_types.TIMESTAMP_LTZ
+ ),
+ parsy.string("TIMESTAMP_TZ").result(lambda: sqltypes.DATETIME(timezone=True)),
+ parsy.string("TIMESTAMPTZ").result(lambda: sqltypes.DATETIME(timezone=True)),
time_type(sqltypes.TIMESTAMP, postgresql.types.TIMESTAMP),
time_type(sqltypes.TIME, postgresql.types.TIME),
)
diff --git a/datafaker/utils.py b/datafaker/utils.py
index e7ce255e..65346268 100644
--- a/datafaker/utils.py
+++ b/datafaker/utils.py
@@ -459,19 +459,12 @@ def underline_error(e: SyntaxError) -> str:
def text_list(items: Iterable[str], empty="") -> str:
"""Concatenate the items with commas and one "and"."""
- item_i = iter(items)
- try:
- last_item = next(item_i)
- except StopIteration:
+ items = list(items)
+ if not items:
return empty
- try:
- so_far = next(item_i)
- except StopIteration:
- return last_item
- for item in item_i:
- so_far += ", " + last_item
- last_item = item
- return so_far + " and " + last_item
+ if len(items) == 1:
+ return items[0]
+ return ", ".join(items[:-1]) + " and " + items[-1]
def gather_from_ast(
diff --git a/docs/source/faq.rst b/docs/source/faq.rst
index a5700f02..82bc9dbb 100644
--- a/docs/source/faq.rst
+++ b/docs/source/faq.rst
@@ -1,19 +1,52 @@
FAQ
===
-Can datafaker work with two different schemas?
+Can DataFaker work with two different schemas?
**********************************************
-datafaker can only work with a single source schema and a single destination schema at a time.
-However, you can choose for the destination schema to have a different name to the source schema by setting the ``DST_SCHEMA`` environment variable.
+DataFaker can only work with a single source schema or a single destination schema at a time.
+It is possible to use the same database for source and destination by choosing a different schema for source and destination,
+for example by setting ``DST_SCHEMA=fake``.
Which DBMSs does datafaker support?
***********************************
-* datafaker most fully supports **PostgresSQL**, which it uses for its end-to-end functional tests.
-* datafaker also supports **MariaDB**, as long as you don't set ``use-asyncio: true`` in your config.
-* datafaker *might* work with **SQLite** but this is largely untested.
-* datafaker may also work with SQL Server.
- To connect to SQL Server, you will need to install `pyodbc `_ and an `ODBC driver `_, after which you should be able to use a DSN setting similar to ``SRC_DSN="mssql+pyodbc://username:password@hostname/dbname?driver=ODBC Driver 18 for SQL Server"``.
+PostgreSQL
+^^^^^^^^^^
-Please open a GitHub issue if you would like to see support for another DBMS.
+Fully supported.
+
+Microsoft SQL Server
+^^^^^^^^^^^^^^^^^^^^
+
+Supported. Microsoft Transact SQL probably works as well.
+You need to install an `ODBC driver `_.
+
+The DSN format is similar to:
+
+``mssql+pyodbc://:@127.0.0.1:1433/?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes``
+
+DuckDB
+^^^^^^
+
+Supported.
+If you want to produce Parquet or CSV files as your output format, the easiest way is via a DuckDB database.
+You do not need to install DuckDB to do this, simply use a destination DSN like ``duckdb:///./fake.db``,
+then use ``datafaker dump-data --output my-csv-folder`` or ``datafaker dump-data --parquet --output my-parquet-folder``.
+
+An in-memory DuckDB database can be used as an empty database that you can augment with parquet files:
+
+``datafaker make-tables --parquet-dir my-parquet-input-folder``
+
+Snowflake
+^^^^^^^^^
+
+Currently experimental support. If the input database includes the type ``TIMEZONE_LTZ`` (or equivalent)
+then the output database must also be a Snowflake database.
+
+Other
+^^^^^
+
+In principle DataFaker can support any database that SqlAlchemy supports.
+However, each new database supported so far has required significant effort,
+so no guarantees can be made for, for example, MySQL or MariaDB.
diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst
index def95218..7c696958 100644
--- a/docs/source/quickstart.rst
+++ b/docs/source/quickstart.rst
@@ -78,6 +78,30 @@ The first job is to read the structure of the source database:
This will create a file called ``orm.yaml``. You should not need to edit this file.
+Troubleshooting: What if my tables do not appear in the orm.yaml file?
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If the tables you expected to see do not all appear in the ``orm.yaml`` file,
+perhaps these are not actual tables, but views.
+
+DataFaker can copy views as though they are tables. If you want this behaviour
+(for example, if you have no access to the underlying tables) you can use
+the ``--views-to-tables`` option:
+
+.. code-block:: console
+
+ $ datafaker make-tables --views-to-tables
+
+This will produce definitions of each view as well as each table.
+This is not a perfect system; DataFaker will guess which columns represent
+primary or foreign keys as this information is not present in views.
+So, after running ``make-tables`` with the ``--view-to-tables`` option,
+please check the ``orm.yaml`` file produced:
+
+* make sure that each table has its correct primary key (DataFaker will have warned you if DataFaker found no or multiple primary keys for any table)
+* make sure that all foreign key columns are correctly marked
+* delete any tables you don't want and have no foreign keys pointing to them (or you could mark them as ignored in the following ``configure-tables`` stage)
+
Configuring table types
-----------------------
diff --git a/examples/omop-mssql/README.md b/examples/omop-mssql/README.md
index 7c0bbfed..64662a35 100644
--- a/examples/omop-mssql/README.md
+++ b/examples/omop-mssql/README.md
@@ -51,8 +51,6 @@ PostgreSQL uses `SERIAL` for autoincrement columns. The code already strips `SER
[datafaker/utils.py:297-309](../../datafaker/utils.py) toggles `connection.autocommit` directly on the DBAPI connection before executing `SET` commands. The availability and behaviour of `autocommit` differs between `psycopg2`, `pyodbc`, and `pymssql`, so this would need testing or an abstraction per driver.
-**Deferred.** `set_db_settings` is only ever called for DuckDB connections (`parquet_dir` source feature); MS-SQL connections never reach it. Fixing the `autocommit` toggle in isolation would also leave the `SET {k} TO {v}` SQL syntax broken for MS-SQL (which requires `SET {k} {v}`). Both issues should be addressed together if `set_db_settings` is ever extended to MS-SQL.
-
## Setup
### 1. Install the MS-SQL Python extras
diff --git a/poetry.lock b/poetry.lock
index a34332f0..81be50d4 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -251,6 +251,44 @@ d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"]
jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
uvloop = ["uvloop (>=0.15.2)"]
+[[package]]
+name = "boto3"
+version = "1.43.106"
+description = "The AWS SDK for Python (Boto3)"
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "boto3-1.43.106-py3-none-any.whl", hash = "sha256:fdf2e304f9e8864b8613591b18b36ee995560c9666033622eda1cb9057866880"},
+ {file = "boto3-1.43.106.tar.gz", hash = "sha256:c11ad4c429a983493ba10014c7af9831a455c2c0eea91c1cefff74530e480277"},
+]
+
+[package.dependencies]
+botocore = ">=1.43.106,<1.44.0"
+jmespath = ">=0.7.1,<2.0.0"
+s3transfer = ">=0.19.0,<0.20.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
+
+[[package]]
+name = "botocore"
+version = "1.43.106"
+description = "Low-level, data-driven core of boto 3."
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "botocore-1.43.106-py3-none-any.whl", hash = "sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78"},
+ {file = "botocore-1.43.106.tar.gz", hash = "sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97"},
+]
+
+[package.dependencies]
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3"
+
+[package.extras]
+crt = ["awscrt (==0.36.0)"]
+
[[package]]
name = "certifi"
version = "2026.7.22"
@@ -262,6 +300,118 @@ files = [
{file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"},
]
+[[package]]
+name = "cffi"
+version = "2.1.1"
+description = "Foreign Function Interface for Python calling C code."
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "cffi-2.1.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be"},
+ {file = "cffi-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b"},
+ {file = "cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004"},
+ {file = "cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9"},
+ {file = "cffi-2.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98"},
+ {file = "cffi-2.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9"},
+ {file = "cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6"},
+ {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf"},
+ {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659"},
+ {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9"},
+ {file = "cffi-2.1.1-cp310-cp310-win32.whl", hash = "sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41"},
+ {file = "cffi-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1"},
+ {file = "cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12"},
+ {file = "cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1"},
+ {file = "cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0"},
+ {file = "cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813"},
+ {file = "cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990"},
+ {file = "cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af"},
+ {file = "cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632"},
+ {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd"},
+ {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a"},
+ {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa"},
+ {file = "cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3"},
+ {file = "cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0"},
+ {file = "cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455"},
+ {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"},
+ {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"},
+ {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"},
+ {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"},
+ {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"},
+ {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"},
+ {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"},
+ {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"},
+ {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"},
+ {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"},
+ {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"},
+ {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"},
+ {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e"},
+ {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c"},
+ {file = "cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6"},
+ {file = "cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971"},
+ {file = "cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c"},
+ {file = "cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125"},
+ {file = "cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264"},
+ {file = "cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3"},
+ {file = "cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2"},
+ {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b"},
+ {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7"},
+ {file = "cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac"},
+ {file = "cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d"},
+ {file = "cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973"},
+ {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c"},
+ {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb"},
+ {file = "cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54"},
+ {file = "cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72"},
+ {file = "cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1"},
+ {file = "cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062"},
+ {file = "cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03"},
+ {file = "cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96"},
+ {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527"},
+ {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13"},
+ {file = "cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c"},
+ {file = "cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48"},
+ {file = "cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836"},
+ {file = "cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3"},
+ {file = "cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2"},
+ {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94"},
+ {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc"},
+ {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29"},
+ {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676"},
+ {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e"},
+ {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f"},
+ {file = "cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4"},
+ {file = "cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e"},
+ {file = "cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5"},
+ {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d"},
+ {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b"},
+ {file = "cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4"},
+ {file = "cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8"},
+ {file = "cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6"},
+ {file = "cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80"},
+ {file = "cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779"},
+ {file = "cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399"},
+ {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688"},
+ {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7"},
+ {file = "cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac"},
+ {file = "cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960"},
+ {file = "cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1"},
+ {file = "cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc"},
+ {file = "cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab"},
+ {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e"},
+ {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358"},
+ {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231"},
+ {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6"},
+ {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94"},
+ {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5"},
+ {file = "cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66"},
+ {file = "cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3"},
+ {file = "cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692"},
+ {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"},
+]
+
+[package.dependencies]
+pycparser = {version = "*", markers = "implementation_name != \"PyPy\""}
+
[[package]]
name = "cfgv"
version = "3.5.0"
@@ -545,6 +695,81 @@ files = [
[package.extras]
dev = ["black (==22.3.0)", "hypothesis (==6.60.0)", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"]
+[[package]]
+name = "cryptography"
+version = "50.0.2"
+description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
+optional = false
+python-versions = "!=3.9.0,!=3.9.1,>=3.9"
+files = [
+ {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"},
+ {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"},
+ {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"},
+ {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"},
+ {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"},
+ {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"},
+ {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"},
+ {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"},
+ {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"},
+ {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"},
+ {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"},
+ {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"},
+ {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"},
+ {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"},
+ {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"},
+ {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"},
+ {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"},
+ {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"},
+ {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"},
+ {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"},
+ {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"},
+ {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"},
+ {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"},
+]
+
+[package.dependencies]
+cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""}
+typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11\""}
+
+[package.extras]
+ssh = ["bcrypt (>=3.1.5)"]
+
[[package]]
name = "dataclasses-json"
version = "0.6.7"
@@ -997,6 +1222,17 @@ MarkupSafe = ">=2.0"
[package.extras]
i18n = ["Babel (>=2.7)"]
+[[package]]
+name = "jmespath"
+version = "1.1.0"
+description = "JSON Matching Expressions"
+optional = false
+python-versions = ">=3.9"
+files = [
+ {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"},
+ {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"},
+]
+
[[package]]
name = "json-schema-for-humans"
version = "1.5.1"
@@ -1756,6 +1992,22 @@ wcwidth = ">=0.3.5"
[package.extras]
tests = ["pytest (>=9)", "pytest-cov", "pytest-lazy-fixtures"]
+[[package]]
+name = "psycopg2"
+version = "2.9.13"
+description = "psycopg2 - Python-PostgreSQL Database Adapter"
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "psycopg2-2.9.13-cp310-cp310-win_amd64.whl", hash = "sha256:7d48416f6a4823ada9b33771085331b842b553df88435701bff5ddb4469905de"},
+ {file = "psycopg2-2.9.13-cp311-cp311-win_amd64.whl", hash = "sha256:d16e7a5f5e400ac51ca953d42255804eff6c8a9650b1a2074f6ca6261d740382"},
+ {file = "psycopg2-2.9.13-cp312-cp312-win_amd64.whl", hash = "sha256:10f7408b34412e8c0d4f8b1565541f1d651b1d00447857e5d8561b38f5c1a738"},
+ {file = "psycopg2-2.9.13-cp313-cp313-win_amd64.whl", hash = "sha256:165e25c1b0e616a1f28080c5c68bd2dc015051d83c90240b2171d3e76ca2b5ff"},
+ {file = "psycopg2-2.9.13-cp314-cp314-win_amd64.whl", hash = "sha256:a6f54fd8e0024f35240866b5dfff9ead2a0dbd33b8096eec438bc6093412842d"},
+ {file = "psycopg2-2.9.13-cp315-cp315-win_amd64.whl", hash = "sha256:0d2fc7eedfaca0586dcf1476454598428d0d8471d3b5cb55f92015a5f9d0af40"},
+ {file = "psycopg2-2.9.13.tar.gz", hash = "sha256:d36784fc2dae69523ba4b79c7d1d1b4d6e83e87836874f111262f4db940b16a6"},
+]
+
[[package]]
name = "psycopg2-binary"
version = "2.9.12"
@@ -1832,6 +2084,17 @@ files = [
{file = "psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c"},
]
+[[package]]
+name = "pycparser"
+version = "3.0"
+description = "C parser in Python"
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"},
+ {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"},
+]
+
[[package]]
name = "pydantic"
version = "1.10.26"
@@ -1917,6 +2180,23 @@ files = [
[package.extras]
windows-terminal = ["colorama (>=0.4.6)"]
+[[package]]
+name = "pyjwt"
+version = "2.15.1"
+description = "JSON Web Token implementation in Python"
+optional = false
+python-versions = ">=3.9"
+files = [
+ {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"},
+ {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"},
+]
+
+[package.dependencies]
+typing_extensions = {version = ">=4.0", markers = "python_version < \"3.11\""}
+
+[package.extras]
+crypto = ["cryptography (>=3.4.0)"]
+
[[package]]
name = "pylint"
version = "3.3.9"
@@ -2034,6 +2314,25 @@ files = [
{file = "pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05"},
]
+[[package]]
+name = "pyopenssl"
+version = "26.4.0"
+description = "Python wrapper module around the OpenSSL library"
+optional = false
+python-versions = ">=3.9"
+files = [
+ {file = "pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c"},
+ {file = "pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7"},
+]
+
+[package.dependencies]
+cryptography = ">=49.0.0,<51"
+typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\""}
+
+[package.extras]
+docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"]
+test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"]
+
[[package]]
name = "pytest"
version = "9.1.1"
@@ -2408,6 +2707,23 @@ sphinx = ["sphinx (>=4.0,<6.0)"]
testing = ["coverage-conditional-plugin (>=0.5)", "coverage[toml] (>=6.0)", "pytest (>=6.0)", "pytest-cov (>=3.0)", "pytest-mock (>=3.7)", "pytest-randomly (>=3.0)", "pytest-sugar (>=0.9.5)"]
toml = ["tomli (>=2.0,<3.0)"]
+[[package]]
+name = "s3transfer"
+version = "0.19.2"
+description = "An Amazon S3 Transfer Manager"
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25"},
+ {file = "s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993"},
+]
+
+[package.dependencies]
+botocore = ">=1.37.4,<2.0a.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
+
[[package]]
name = "scramp"
version = "1.4.12"
@@ -2474,6 +2790,102 @@ files = [
{file = "snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260"},
]
+[[package]]
+name = "snowflake-connector-python"
+version = "4.8.0"
+description = "Snowflake Connector for Python"
+optional = false
+python-versions = ">=3.10"
+files = [
+ {file = "snowflake_connector_python-4.8.0-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:5ac7214337e2ca4797a63b12e2d6778b85b95bacbb1e8892e8ef5cdb2696bcdb"},
+ {file = "snowflake_connector_python-4.8.0-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:e3de20df18a0279cc913ecd073b79eae860fa00e3d0e8cb9cfb276ecf85fe319"},
+ {file = "snowflake_connector_python-4.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a887838e59afa9d6d5f0fb52f55fcdb8726987acce646daa9aa3c698a3c99690"},
+ {file = "snowflake_connector_python-4.8.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b95cec73e136a3a373bf19bc19ecc5707557ae9df02674665240cfd46ea78820"},
+ {file = "snowflake_connector_python-4.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:3566b6884073dd1c36905c875aa3013991ae814a75f539c6ea1f2e8a1f72dd41"},
+ {file = "snowflake_connector_python-4.8.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:3b38204c9c8a9c985cdf36df4eaa16f7d0b8a14f86410f39ab7b49794df4cf03"},
+ {file = "snowflake_connector_python-4.8.0-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:3fdea6d45bd22142293ece1224f2ad3a3c169a2f4a768d4be828faf09a640b2b"},
+ {file = "snowflake_connector_python-4.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bf2304eedfd94694291a71c96cddb4490e474d2c232625f528813f5caa79c098"},
+ {file = "snowflake_connector_python-4.8.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf8d819fd2c943d9be9d521fb6259b28dca05844e6a4d585a9f6e287141ae6dd"},
+ {file = "snowflake_connector_python-4.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:1ac5bc235c9f1124bfa1b14413f340ef2bdf7c28746af37280eae3ad7877afa8"},
+ {file = "snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:2fde1216c4c1e4015ad7e39b7b39b6a44a29551f4d7a3dfc64b0905e66dc24ac"},
+ {file = "snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c40a058f87dfbca1e65122932afb41c42cb9c3b7cb12826204527b41dd0f39ef"},
+ {file = "snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2a3cbcd3e6c4a449165f78bb331e1570ce6026cc21a2673313b4912d80745c0"},
+ {file = "snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c80df2fc5e16f3a49e6d16b1cdca0358308c1db2eae22d82aa7c038c80cc913"},
+ {file = "snowflake_connector_python-4.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:b9bddddc713aaa20dc5ff5635896f66e50c0fed88b6a905b088e727d046e361d"},
+ {file = "snowflake_connector_python-4.8.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebcbad6b16f8228f28bf62b443b6f111d11bceea2612efa2004eccb827923e6b"},
+ {file = "snowflake_connector_python-4.8.0-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:1cc888b81b709781eb855bbee062f06a9f52427ab8f9a39a31f39de475e39465"},
+ {file = "snowflake_connector_python-4.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2a59396eade5b7b18cad0ac1fa2882a5ddfa5153e56e16c9d3d4a9d5e2d92332"},
+ {file = "snowflake_connector_python-4.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f978f897f85fd39cc8f768971c4667f8ca0278ea369373dfc459ae008c9472a5"},
+ {file = "snowflake_connector_python-4.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:131db582e383b57af0b345f212c9afba87637e8ef68494b9b4282e6713030dc9"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:7a1566b0fcae33f2c7ec1bbd09965d8045a9388e94d333a88d03ddcfb6d4550a"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:8db3eea888247e4b41b55b8b419613eb1c4c76ec4984341543f849bf9532ae1b"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:39fe1794c63508fb25fda1a57cdf00293dc0630b32d4cda86d5bed2da09eb256"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:43583fb372a3bb698c2ce4717d2598e5452b40543a59b8095146a98f8ecd7408"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:6ba170f3584a425a1a5eff0e53ed656d45db9ef513d9232789cf58c5254b8194"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314t-macosx_15_0_arm64.whl", hash = "sha256:03e248f510ae82a3b190146c060e865905e5e07df6cf3e365ede66c67de136b2"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:7612ef5b4a93438fec6fff0e0d2541302f29001d3e77b861140d00d9085c0dce"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bccf23cd780c85bfa74982f286d2aacd663ba0135ba7441a13e98cabdb2def1e"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da28999b3d6cfc4860c8c537cfb02968e9784f3ec8fe50d92a7348133ea75362"},
+ {file = "snowflake_connector_python-4.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6c0d7be79b333c1fa85d2860928f286a11abe5789d007dbc6d940313a744f8af"},
+ {file = "snowflake_connector_python-4.8.0.tar.gz", hash = "sha256:1bf40aa8684eb0769fedeb53a702b274472571812835d5a03fb9dcf9254333b9"},
+]
+
+[package.dependencies]
+asn1crypto = ">0.24.0,<2.0.0"
+boto3 = ">=1.24"
+botocore = ">=1.24"
+certifi = ">=2024.7.4"
+charset_normalizer = ">=2,<4"
+cryptography = ">=46.0.5"
+filelock = ">=3.5,<4"
+idna = ">=3.7,<4"
+packaging = "*"
+platformdirs = ">=2.6.0,<5.0.0"
+pyjwt = ">=2.10.1,<3.0.0"
+pyOpenSSL = ">=25.3.0"
+pytz = "*"
+requests = ">=2.32.4,<3.0.0"
+sortedcontainers = ">=2.4.0"
+tomlkit = "*"
+typing_extensions = ">=4.3,<5"
+
+[package.extras]
+azure = ["azure-identity (>=1.16)"]
+boto = ["boto3 (>=1.24)", "botocore (>=1.24)"]
+development = ["Cython (>=3.1)", "azure-identity (>=1.16)", "coverage", "mitmproxy (>=12.0.0)", "more-itertools", "numpy (<=2.4.3)", "pendulum (!=2.1.1)", "pexpect", "pytest (>=8.4,<9)", "pytest-asyncio (>=1.4.0,<2)", "pytest-cov", "pytest-rerunfailures (<16.0)", "pytest-timeout", "pytest-xdist", "pytzdata", "responses"]
+pandas = ["pandas (>=1.0.0,<3.0.0)", "pandas (>=2.1.2,<3.0.0)", "pyarrow (>=14.0.1)", "pyarrow (>=14.0.1,<24)"]
+secure-local-storage = ["keyring (>=23.1.0,<26.0.0)"]
+
+[[package]]
+name = "snowflake-sqlalchemy"
+version = "1.11.1"
+description = "Snowflake SQLAlchemy Dialect"
+optional = false
+python-versions = ">=3.8"
+files = [
+ {file = "snowflake_sqlalchemy-1.11.1-py3-none-any.whl", hash = "sha256:b64ab947469bf25b3df3f88c5b92f17a938c9d47b5b09035551614f3ff1f44d9"},
+ {file = "snowflake_sqlalchemy-1.11.1.tar.gz", hash = "sha256:942b203de0404b79550c4098ca851aee2b0b22f695426b073e58d71772e38d51"},
+]
+
+[package.dependencies]
+snowflake-connector-python = "<5.0.0"
+sqlalchemy = ">=1.4.19"
+
+[package.extras]
+development = ["alembic", "numpy", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytz", "setuptools", "syrupy"]
+pandas = ["snowflake-connector-python[pandas]"]
+
+[[package]]
+name = "sortedcontainers"
+version = "2.4.0"
+description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set"
+optional = false
+python-versions = "*"
+files = [
+ {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"},
+ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"},
+]
+
[[package]]
name = "sphinx"
version = "5.3.0"
@@ -3174,4 +3586,4 @@ docs = ["sphinx-rtd-theme", "sphinxcontrib-napoleon"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.14"
-content-hash = "32e39aeac70fde68398a8a5d467c29499cd74d177beda122a19f4641689f71c3"
+content-hash = "5762718d243ff3e673d19efc74904d0302ac47e444598edd25132b10c2498673"
diff --git a/pyproject.toml b/pyproject.toml
index aa31415b..1d44b2b7 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,6 +40,8 @@ fastparquet = "^2024.11.0"
duckdb-sqlalchemy = "^1.5.2.2"
ty = "^0.0.43"
colorama = "^0.4.6"
+snowflake-sqlalchemy = "^1.11.1"
+psycopg2 = "^2.9.13"
[tool.poetry.group.dev.dependencies]
isort = "^5.10.1"
diff --git a/tests/examples/date.sql b/tests/examples/date.sql
index b8e99be7..2f531718 100644
--- a/tests/examples/date.sql
+++ b/tests/examples/date.sql
@@ -47,3 +47,8 @@ INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:0
INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL, 1);
INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL, 1);
INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL, NULL);
+
+CREATE VIEW public.happening_view (id, name, person_id)
+ AS SELECT id, name, person_id FROM public.happening;
+
+ALTER VIEW public.happening_view OWNER TO postgres;
diff --git a/tests/examples/duckdb/date/other_person.parquet b/tests/examples/duckdb/date/other_person.parquet
new file mode 100644
index 00000000..0012995d
Binary files /dev/null and b/tests/examples/duckdb/date/other_person.parquet differ
diff --git a/tests/examples/duckdb/config.yaml b/tests/examples/duckdb/instrument/config.yaml
similarity index 100%
rename from tests/examples/duckdb/config.yaml
rename to tests/examples/duckdb/instrument/config.yaml
diff --git a/tests/examples/duckdb/manufacturer.parquet b/tests/examples/duckdb/instrument/manufacturer.parquet
similarity index 100%
rename from tests/examples/duckdb/manufacturer.parquet
rename to tests/examples/duckdb/instrument/manufacturer.parquet
diff --git a/tests/examples/duckdb/model.parquet b/tests/examples/duckdb/instrument/model.parquet
similarity index 100%
rename from tests/examples/duckdb/model.parquet
rename to tests/examples/duckdb/instrument/model.parquet
diff --git a/tests/examples/duckdb/orm.yaml b/tests/examples/duckdb/instrument/orm.yaml
similarity index 100%
rename from tests/examples/duckdb/orm.yaml
rename to tests/examples/duckdb/instrument/orm.yaml
diff --git a/tests/examples/duckdb/player.parquet b/tests/examples/duckdb/instrument/player.parquet
similarity index 100%
rename from tests/examples/duckdb/player.parquet
rename to tests/examples/duckdb/instrument/player.parquet
diff --git a/tests/examples/duckdb/signature_model.parquet b/tests/examples/duckdb/instrument/signature_model.parquet
similarity index 100%
rename from tests/examples/duckdb/signature_model.parquet
rename to tests/examples/duckdb/instrument/signature_model.parquet
diff --git a/tests/examples/duckdb/spec.csv b/tests/examples/duckdb/instrument/spec.csv
similarity index 100%
rename from tests/examples/duckdb/spec.csv
rename to tests/examples/duckdb/instrument/spec.csv
diff --git a/tests/examples/duckdb/string.parquet b/tests/examples/duckdb/instrument/string.parquet
similarity index 100%
rename from tests/examples/duckdb/string.parquet
rename to tests/examples/duckdb/instrument/string.parquet
diff --git a/tests/examples/duckdb/measurement/type=1/third_value=no/data_0.parquet b/tests/examples/duckdb/measurement/type=1/third_value=no/data_0.parquet
new file mode 100644
index 00000000..0973fef9
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=1/third_value=no/data_0.parquet differ
diff --git a/tests/examples/duckdb/measurement/type=1/third_value=yes/data_0.parquet b/tests/examples/duckdb/measurement/type=1/third_value=yes/data_0.parquet
new file mode 100644
index 00000000..3e85dc1d
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=1/third_value=yes/data_0.parquet differ
diff --git a/tests/examples/duckdb/measurement/type=2/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet b/tests/examples/duckdb/measurement/type=2/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet
new file mode 100644
index 00000000..670f3802
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=2/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet differ
diff --git a/tests/examples/duckdb/measurement/type=3/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet b/tests/examples/duckdb/measurement/type=3/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet
new file mode 100644
index 00000000..c1d749f3
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=3/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet differ
diff --git a/tests/examples/duckdb/measurement/type=4/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet b/tests/examples/duckdb/measurement/type=4/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet
new file mode 100644
index 00000000..3872d227
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=4/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet differ
diff --git a/tests/examples/duckdb/measurement/type=5/third_value=fish/data_0.parquet b/tests/examples/duckdb/measurement/type=5/third_value=fish/data_0.parquet
new file mode 100644
index 00000000..0152aaf6
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=5/third_value=fish/data_0.parquet differ
diff --git a/tests/examples/duckdb/measurement/type=5/third_value=fowl/data_0.parquet b/tests/examples/duckdb/measurement/type=5/third_value=fowl/data_0.parquet
new file mode 100644
index 00000000..b4f4660b
Binary files /dev/null and b/tests/examples/duckdb/measurement/type=5/third_value=fowl/data_0.parquet differ
diff --git a/tests/test_dump.py b/tests/test_dump.py
index d6a838a1..55811890 100644
--- a/tests/test_dump.py
+++ b/tests/test_dump.py
@@ -101,7 +101,7 @@ class EndToEndParquetTestCase(DatafakerTestCase):
"""Read in parquet, make some generators, output parquet."""
database_type = DuckTestDb
- examples_dir = Path("examples/duckdb")
+ examples_dir = Path("examples/duckdb/instrument")
def set_working_dir(self) -> None:
"""Change to our working directory."""
@@ -114,9 +114,10 @@ def setUp(self) -> None:
super().setUp()
# Grab all the files
self.parquet_dir = Path(tempfile.mkdtemp("parq"))
- example_dir = self.get_abs_example_dir()
- for fname in os.listdir(example_dir):
- shutil.copy(example_dir / fname, self.parquet_dir / fname)
+ example_dir = Path(self.get_abs_example_dir())
+ for child in example_dir.iterdir():
+ if child.is_file():
+ shutil.copy(example_dir / child, self.parquet_dir / child.name)
self.start_dir = os.getcwd()
self.set_working_dir()
@@ -133,22 +134,31 @@ def make_orm_yaml(self, runner: CliRunner) -> None:
"make-tables",
"--parquet-dir",
str(self.parquet_dir),
- "--orm-file",
- "orm_auto.yaml",
],
)
self.assertSuccess(result)
- # Fix up the orm.yaml; the dates might not have types set
- with Path("orm_auto.yaml").open(encoding="utf-8") as orm_fh:
+ with Path("orm.yaml").open(encoding="utf-8") as orm_fh:
orm = yaml.load(orm_fh, yaml.SafeLoader)
+ # Let's check some of the types, primary and foreign keys
t = orm["tables"]
+ self.assertEqual(
+ t["signature_model.parquet"]["columns"]["player_id"]["type"], "INTEGER"
+ )
+ self.assertTrue(t["manufacturer.parquet"]["columns"]["id"]["primary"])
+ self.assertFalse(
+ t["manufacturer.parquet"]["columns"]["name"].get("primary", False)
+ )
+ self.assertEqual(
+ t["signature_model.parquet"]["columns"]["player_id"]["foreign_keys"],
+ ["player.parquet.id"],
+ )
+ self.assertEqual(
+ t["signature_model.parquet"]["columns"]["based_on"]["type"], "INTEGER"
+ )
+ # DATETIMEs cannot be detected
t["manufacturer.parquet"]["columns"]["founded"]["type"] = "DATETIME"
t["model.parquet"]["columns"]["introduced"]["type"] = "DATETIME"
- t["signature_model.parquet"]["columns"]["player_id"]["type"] = "INTEGER"
- t["signature_model.parquet"]["columns"]["player_id"]["foreign_keys"] = [
- "player.parquet.id"
- ]
- t["signature_model.parquet"]["columns"]["based_on"]["type"] = "INTEGER"
+ # "based_on" doesn't give the foreign key guesser anything to go on
t["signature_model.parquet"]["columns"]["based_on"]["foreign_keys"] = [
"model.parquet.id"
]
diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py
index dbf0182a..64c033f5 100644
--- a/tests/test_evaluators_proposal_ranking.py
+++ b/tests/test_evaluators_proposal_ranking.py
@@ -1,6 +1,8 @@
"""Unit tests for datafaker.evaluators.proposal_ranking."""
from typing import Any
+from sqlalchemy import Column, MetaData, Table, Text
+
from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation
from datafaker.evaluators.proposal_ranking import (
ProposalRanking,
@@ -227,7 +229,8 @@ def test_keyword_boost_can_flip_the_recommendation(self) -> None:
def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None:
"""ChoiceProposer's score is discounted by real_uniqueness x copy_fraction."""
- choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1])
+ table = Table("t", MetaData(), Column("c", Text()))
+ choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1])
results = [
make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer),
make_result(overall_score=0.5, copy_fraction=0.0),
@@ -307,7 +310,8 @@ def test_combined_score_is_clamped_to_one(self) -> None:
def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None:
"""A heavily-discounted best-fit candidate triggers the fallback warning."""
- choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1])
+ table = Table("t", MetaData(), Column("c", Text()))
+ choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1])
results = [
# Fits almost perfectly but is a resampler on a near-unique column:
# gets discounted to (near) zero.
diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py
index dbd6415f..89191868 100644
--- a/tests/test_functional_mssql.py
+++ b/tests/test_functional_mssql.py
@@ -17,6 +17,7 @@
from tempfile import mkstemp
import yaml
+from sqlalchemy import Column, MetaData, Table, Text
from sqlalchemy import create_engine as sa_create_engine
from sqlalchemy import text
from sqlalchemy.dialects import mssql as mssql_dialect # noqa: PLC0415
@@ -52,7 +53,9 @@ def setUp(self) -> None:
(self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True)
with os.fdopen(self.orm_fd, "w", encoding="utf-8") as fh:
fh.write(
- make_tables_file(self.dsn, self.schema_name, engine=self.sync_engine)
+ make_tables_file(
+ self.dsn, self.schema_name, False, engine=self.sync_engine
+ )
)
def tearDown(self) -> None:
@@ -119,9 +122,9 @@ def test_dialect_rand(self) -> None:
"""ChoiceProposer compiles its query with NEWID() not RANDOM() for mssql."""
dialect = mssql_dialect.dialect()
+ table = Table("manufacturer", MetaData(), Column("name", Text()))
proposer = ZipfChoiceProposer(
- table_name="manufacturer",
- column_name="name",
+ column=table.c.name,
values=["Blender", "Gibbs"],
counts=[5, 5],
sample_count=2,
diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py
index 4ddc21bb..99261c12 100644
--- a/tests/test_generators_dialect.py
+++ b/tests/test_generators_dialect.py
@@ -4,7 +4,7 @@
import unittest.mock
from unittest.mock import MagicMock
-from sqlalchemy import Column, Integer, MetaData, Select, Table, literal_column
+from sqlalchemy import Column, Integer, MetaData, Select, Table, Text, literal_column
from sqlalchemy.dialects import mssql, postgresql
from sqlalchemy.types import DateTime
@@ -146,9 +146,13 @@ class TestChoiceGeneratorStoredQuery(DatafakerTestCase):
"""ChoiceGenerator._query is compiled to dialect-correct SQL at construction time."""
def _make_gen(self, dialect, sample_count=None, suppress_count=0):
+ table = Table(
+ "patient",
+ MetaData(),
+ Column("gender", Text()),
+ )
return ZipfChoiceProposer(
- table_name="patient",
- column_name="gender",
+ column=table.c.gender,
values=["M", "F"],
counts=[70, 30],
sample_count=sample_count,
diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py
index 0a64e526..de84689a 100644
--- a/tests/test_interactive_generators.py
+++ b/tests/test_interactive_generators.py
@@ -240,9 +240,9 @@ def test_set_generator_choice(self) -> None:
gc.config["src-stats"][0]["query"],
(
"SELECT _counted.value \n"
- f'FROM (SELECT "{column}" AS value, count("{column}") AS count \n'
+ f"FROM (SELECT {table}.{column} AS value, count({table}.{column}) AS count \n"
f"FROM {table} \n"
- f'WHERE "{column}" IS NOT NULL GROUP BY "{column}") '
+ f"WHERE {table}.{column} IS NOT NULL GROUP BY {table}.{column}) "
"AS _counted ORDER BY _counted.count DESC"
),
)
diff --git a/tests/test_interactive_missingness.py b/tests/test_interactive_missingness.py
index c1da7c54..747f9e0f 100644
--- a/tests/test_interactive_missingness.py
+++ b/tests/test_interactive_missingness.py
@@ -1,7 +1,8 @@
""" Tests for the configure-missingness command. """
import random
from collections.abc import MutableMapping
-from typing import Any
+from pathlib import Path
+from typing import Any, Iterable
from sqlalchemy import select
@@ -10,6 +11,15 @@
from tests.utils import GeneratesDBTestCase, RequiresDBTestCase, TestDbCmdMixin
+def count_nonnull(it: Iterable):
+ """Count the number of non-null values in an iterable."""
+ n = 0
+ for v in it:
+ if v is not None:
+ n += 1
+ return n
+
+
class MockMissingnessCmd(MissingnessCmd, TestDbCmdMixin):
"""MissingnessCmd but mocked"""
@@ -107,3 +117,39 @@ def test_create_with_missingness(self) -> None:
patterns.add(p + b)
# all pattern possibilities should be present
self.assertSetEqual(patterns, {0, 1, 2, 3})
+
+
+class ConfigureMissingnessEndToEndTests(ConfigureMissingnessTestsWithGeneration):
+ """Test `configure-missing` end-to-end."""
+
+ use_temporary_cwd = True
+ copy_files = ["row_generators.py", "story_generators.py"]
+ copy_from_directory = Path("examples")
+
+ def test_missingness_end_to_end(self) -> None:
+ """Test that if an interval end is applicable it gets proposed and works."""
+ table = "signature_model"
+ with self._get_cmd({}) as mc:
+ mc.do_next(table)
+ mc.do_sampled("")
+ mc.reset()
+ mc.do_quit("")
+ self.generate_data(mc.config, num_passes=100)
+ self.assertIs(
+ type(
+ mc.config["tables"][table]["missingness_generators"][0]["columns"][
+ 0
+ ]
+ ),
+ str,
+ )
+ assert self.dst_engine is not None
+ with self.dst_engine.connect() as conn:
+ stmt = select(self.metadata.tables[table])
+ rows = conn.execute(stmt).mappings().fetchall()
+ name_count = count_nonnull(r.name for r in rows)
+ player_id_count = count_nonnull(r.player_id for r in rows)
+ based_on_count = count_nonnull(r.based_on for r in rows)
+ self.assertEqual(name_count, 100)
+ self.assertAlmostEqual(player_id_count, 50, delta=20)
+ self.assertAlmostEqual(based_on_count, 70, delta=20)
diff --git a/tests/test_main.py b/tests/test_main.py
index 02878b14..ab9819f4 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -111,7 +111,10 @@ def test_make_tables(
)
mock_make_tables_file.assert_called_once_with(
- "postgresql://suser:spassword@shost:5432/sdbname", None, None
+ "postgresql://suser:spassword@shost:5432/sdbname",
+ None,
+ views_to_tables=False,
+ parquet_dir=None,
)
mock_path.return_value.write_text.assert_called_once_with(
"some text", encoding="utf-8"
@@ -176,6 +179,8 @@ def test_make_tables_with_force_enabled(
for force_option in ["--force", "-f"]:
with self.subTest(f"Using option {force_option}"):
+ mock_make_tables.reset_mock()
+ mock_path.reset_mock()
result: Result = runner.invoke(
app,
[
@@ -188,16 +193,14 @@ def test_make_tables_with_force_enabled(
mock_make_tables.assert_called_once_with(
mock_get_settings.return_value.src_dsn,
mock_get_settings.return_value.src_schema,
- None,
+ views_to_tables=False,
+ parquet_dir=None,
)
mock_path.return_value.write_text.assert_called_once_with(
mock_tables_output, encoding="utf-8"
)
self.assertSuccess(result)
- mock_make_tables.reset_mock()
- mock_path.reset_mock()
-
@patch("datafaker.main.Path")
@patch("datafaker.settings.get_settings")
def test_incorrect_dialect_causes_nice_error_message(
diff --git a/tests/test_make.py b/tests/test_make.py
index 9327ea6a..a2c70998 100644
--- a/tests/test_make.py
+++ b/tests/test_make.py
@@ -3,6 +3,7 @@
import tempfile
from pathlib import Path
from typing import Any
+from unittest import SkipTest
from unittest.mock import MagicMock, patch
import pandas as pd
@@ -20,12 +21,136 @@
from sqlalchemy.dialects.mysql.types import INTEGER
from sqlalchemy.dialects.postgresql import UUID
+from datafaker.db_utils import create_db_engine, get_sync_engine
from datafaker.make import (
_get_default_generator,
_get_provider_for_column,
make_src_stats,
+ make_tables_file,
)
-from tests.utils import DatafakerTestCase, GeneratesDBTestCase, RequiresDBTestCase
+from tests.utils import (
+ DatafakerTestCase,
+ GeneratesDBTestCase,
+ MsSqlTestDb,
+ PostgresTestDb,
+ RequiresDBTestCase,
+)
+
+
+class TestSerialize(DatafakerTestCase):
+ """Unit tests for serialize_metadata."""
+
+ database_type = PostgresTestDb
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ super().setUpClass()
+ skip_msg = cls.database_type.skip()
+ if skip_msg:
+ raise SkipTest(skip_msg)
+ cls.database_type.setup()
+
+ def setUp(self) -> None:
+ super().setUp()
+ self.database = self.database_type()
+ self.database.open()
+ self.database.run_sql(self.get_abs_example_dir() / "date.sql")
+ self.engine = create_db_engine(self.database.get_dsn("date_tables"), None)
+
+ def test_metadata_to_dict(self) -> None:
+ """Test metadata_to_dict."""
+ orm_yaml = make_tables_file(
+ self.database.get_dsn("date_tables"),
+ schema_name=None,
+ views_to_tables=False,
+ parquet_dir=None,
+ engine=get_sync_engine(self.engine),
+ )
+ orm = yaml.load(orm_yaml, yaml.Loader)
+ table_names = set(orm["tables"].keys())
+ self.assert_subset({"person", "happening", "happening_view"}, table_names)
+ happening_columns = orm["tables"]["happening"]["columns"]
+ happening_column_names = set(happening_columns.keys())
+ self.assert_subset(
+ {"id", "name", "comment", "person_id"},
+ happening_column_names,
+ )
+ self.assertEqual(happening_columns["person_id"]["type"], "INTEGER")
+ self.assertFalse(happening_columns["name"]["primary"])
+ self.assertTrue(happening_columns["id"]["primary"])
+
+ def test_views_to_dict(self) -> None:
+ """Test views_to_tables."""
+ orm_yaml = make_tables_file(
+ self.database.get_dsn("date_tables"),
+ schema_name=None,
+ views_to_tables=True,
+ parquet_dir=None,
+ engine=get_sync_engine(self.engine),
+ )
+ orm = yaml.load(orm_yaml, yaml.Loader)
+ table_names = set(orm["tables"].keys())
+ self.assert_subset({"person", "happening", "happening_view"}, table_names)
+ happening_columns = orm["tables"]["happening_view"]["columns"]
+ happening_column_names = set(happening_columns.keys())
+ self.assert_subset(
+ {"id", "name", "person_id"},
+ happening_column_names,
+ )
+ self.assertEqual(happening_columns["person_id"]["type"], "INTEGER")
+ self.assertFalse(happening_columns["name"]["primary"])
+
+ def test_views_to_dict_guess_keys(self) -> None:
+ """Test views_to_tables with foreign keys and primary keys guessed."""
+ orm_yaml = make_tables_file(
+ self.database.get_dsn("date_tables"),
+ schema_name=None,
+ views_to_tables=True,
+ parquet_dir=None,
+ engine=get_sync_engine(self.engine),
+ )
+ orm = yaml.load(orm_yaml, yaml.Loader)
+ view_columns = orm["tables"]["happening_view"]["columns"]
+ self.assertTrue(view_columns["id"]["primary"])
+ self.assertEqual(view_columns["person_id"]["foreign_keys"], ["person.id"])
+
+ def test_only_guess_keys_in_views_and_parquet(self) -> None:
+ """Test view_to_tables guesses keys only from parquet or views."""
+ parquet_dir = self.get_abs_example_dir() / "duckdb" / "date"
+ orm_yaml = make_tables_file(
+ self.database.get_dsn("date_tables"),
+ schema_name=None,
+ views_to_tables=True,
+ parquet_dir=parquet_dir,
+ engine=get_sync_engine(self.engine),
+ )
+ orm = yaml.load(orm_yaml, yaml.Loader)
+ # Test view PK and FK to tables
+ view_columns = orm["tables"]["happening_view"]["columns"]
+ self.assertTrue(view_columns["id"]["primary"])
+ self.assertFalse(view_columns["id"].get("nullable", False))
+ self.assertEqual(view_columns["person_id"]["foreign_keys"], ["person.id"])
+ # Test parquet PK and parquet FK to table:
+ # (other_person.happening_id -> happening.id)
+ self.assertIn("other_person.parquet", orm["tables"])
+ parquet_columns = orm["tables"]["other_person.parquet"]["columns"]
+ self.assertTrue(parquet_columns["id"]["primary"])
+ self.assertFalse(parquet_columns["id"].get("nullable", False))
+ self.assertEqual(
+ parquet_columns["happening_id"]["foreign_keys"], ["happening.id"]
+ )
+ # Test that table does not get updated:
+ # (happening.other_person_id should stay as FK to person.id)
+ happening_columns = orm["tables"]["happening"]["columns"]
+ self.assertEqual(
+ happening_columns["other_person_id"]["foreign_keys"], ["person.id"]
+ )
+
+
+class TestSerializeMsSql(TestSerialize):
+ """Unit tests for serialize_metadata with an MS SQL database."""
+
+ database_type = MsSqlTestDb
class TestGetDefaultGenerator(DatafakerTestCase):
diff --git a/tests/test_noninteractive_generators.py b/tests/test_noninteractive_generators.py
index 9dee1166..2c994da4 100644
--- a/tests/test_noninteractive_generators.py
+++ b/tests/test_noninteractive_generators.py
@@ -7,7 +7,7 @@
from datafaker.interactive import update_config_generators
from datafaker.interactive.base import DbCmd
from datafaker.utils import get_property
-from tests.utils import RequiresDBTestCase
+from tests.utils import DuckTestDb, RequiresDBTestCase
class NonInteractiveTests(RequiresDBTestCase):
@@ -20,9 +20,14 @@ class NonInteractiveTests(RequiresDBTestCase):
schema_name = "public"
@patch("datafaker.interactive.Path")
- @patch(
- "datafaker.interactive.csv.reader",
- return_value=iter(
+ @patch("datafaker.interactive.csv.reader")
+ def test_non_interactive_configure_generators(
+ self, mock_csv_reader: MagicMock, _mock_path: MagicMock
+ ) -> None:
+ """
+ test that we can set generators from a CSV file
+ """
+ mock_csv_reader.return_value = iter(
[
["observation", "type", "dist_gen.weighted_choice [sampled]"],
[
@@ -40,14 +45,7 @@ class NonInteractiveTests(RequiresDBTestCase):
],
["observation", "third_value", "dist_gen.weighted_choice"],
]
- ),
- )
- def test_non_interactive_configure_generators(
- self, _mock_csv_reader: MagicMock, _mock_path: MagicMock
- ) -> None:
- """
- test that we can set generators from a CSV file
- """
+ )
config: MutableMapping[str, Any] = {}
spec_csv = MagicMock()
spec_csv.open.return_value.__enter__.return_value = "mock spec.csv file"
@@ -76,9 +74,14 @@ def test_non_interactive_configure_generators(
)
@patch("datafaker.interactive.Path")
- @patch(
- "datafaker.interactive.csv.reader",
- return_value=iter(
+ @patch("datafaker.interactive.csv.reader")
+ def test_non_interactive_configure_null_partitioned(
+ self, mock_csv_reader: MagicMock, _mock_path: MagicMock
+ ) -> None:
+ """
+ test that we can set multi-column generators from a CSV file
+ """
+ mock_csv_reader.return_value = iter(
[
[
"observation",
@@ -86,14 +89,7 @@ def test_non_interactive_configure_generators(
"null-partitioned grouped_multivariate_lognormal",
],
]
- ),
- )
- def test_non_interactive_configure_null_partitioned(
- self, _mock_csv_reader: MagicMock, _mock_path: MagicMock
- ) -> None:
- """
- test that we can set multi-column generators from a CSV file
- """
+ )
config: MutableMapping[str, Any] = {}
spec_csv = MagicMock()
spec_csv.open.return_value.__enter__.return_value = "mock spec.csv file"
@@ -132,26 +128,24 @@ def test_non_interactive_configure_null_partitioned(
)
@patch("datafaker.interactive.Path")
- @patch(
- "datafaker.interactive.csv.reader",
- return_value=iter(
- [
- [
- "observation",
- "type first_value second_value third_value",
- "null-partitioned grouped_multivariate_lognormal",
- ],
- ]
- ),
- )
+ @patch("datafaker.interactive.csv.reader")
def test_non_interactive_configure_null_partitioned_where_existing_merges(
- self, _mock_csv_reader: MagicMock, _mock_path: MagicMock
+ self, mock_csv_reader: MagicMock, _mock_path: MagicMock
) -> None:
"""
test that we can set multi-column generators from a CSV file,
but where there are already multi-column generators configured
that will have to be unmerged.
"""
+ mock_csv_reader.return_value = iter(
+ [
+ [
+ "observation",
+ "type first_value second_value third_value",
+ "null-partitioned grouped_multivariate_lognormal",
+ ],
+ ]
+ )
config = {
"tables": {
"observation": {
@@ -225,3 +219,9 @@ def test_non_interactive_configure_null_partitioned_where_existing_merges(
),
'"grouped_multivariate_lognormal"',
)
+
+
+class NonInteractiveTestsDuckDB(NonInteractiveTests):
+ """DuckDB version of NonInteractiveTests."""
+
+ database_type = DuckTestDb
diff --git a/tests/test_parquet2orm.py b/tests/test_parquet2orm.py
index f4564a34..2ca28013 100644
--- a/tests/test_parquet2orm.py
+++ b/tests/test_parquet2orm.py
@@ -6,15 +6,14 @@
from collections.abc import Iterable
from pathlib import Path
from typing import Any
-from unittest import TestCase
from unittest.mock import MagicMock, patch
import pandas as pd
from sqlalchemy import text
-from typer import Exit
from datafaker.db_utils import create_db_engine, get_sync_engine
-from datafaker.parquet2orm import get_parquet_orm
+from datafaker.parquet2orm import add_guessed_keys, get_initial_parquet_orm
+from tests.utils import DatafakerTestCase
class HasValues:
@@ -37,7 +36,7 @@ def __repr__(self) -> str:
return f"HasValues<{self.value_set}>"
-class ParquetDirTestCase(TestCase):
+class ParquetDirTestCase(DatafakerTestCase):
"""Base class providing a temporary directory for parquet files."""
def setUp(self) -> None:
@@ -83,8 +82,9 @@ def test_can_infer_column_type(self) -> None:
}
}
self.write_parquet(data)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, set(data.keys()))
self.assertSetEqual(set(orm.keys()), set(data.keys()))
self.assertIn("columns", orm["fruit.parquet"])
self.assertSetEqual(
@@ -113,8 +113,9 @@ def test_can_infer_primary_key(self, mock_logger: MagicMock) -> None:
}
}
self.write_parquet(data)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, set(data.keys()))
self.assertSetEqual(set(orm.keys()), set(data.keys()))
primary_keys = {
name
@@ -126,7 +127,7 @@ def test_can_infer_primary_key(self, mock_logger: MagicMock) -> None:
mock_logger.warning.assert_called_once_with(
"Found multiple likely primary keys for table %s: %s",
"fruit.parquet",
- HasValues({"fruit_id", "FruitKey"}),
+ "FruitKey, fruit_id",
)
def test_can_infer_foreign_key(self) -> None:
@@ -140,8 +141,9 @@ def test_can_infer_foreign_key(self) -> None:
"seed.parquet": {"seed_id": [1, 2], "name": ["orange pip", "grape seed"]},
}
self.write_parquet(data)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, set(data.keys()))
self.assertSetEqual(set(orm.keys()), set(data.keys()))
self.assertNotIn("foreign_keys", orm["fruit.parquet"]["columns"]["fruit_id"])
self.assertNotIn("foreign_keys", orm["fruit.parquet"]["columns"]["name"])
@@ -177,8 +179,9 @@ def test_partitioned_directory_is_one_table(self) -> None:
"visit/yr=__MISSING__/p00001.parquet": {"visit_id": [3], "ward": ["c"]},
}
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, {"visit"})
self.assertSetEqual(set(orm.keys()), {"visit"})
cols = orm["visit"]["columns"]
self.assertSetEqual(set(cols.keys()), {"visit_id", "ward", "yr"})
@@ -189,7 +192,9 @@ def test_partitioned_directory_is_one_table(self) -> None:
def test_partition_key_type_is_not_a_warning(self, mock_logger: MagicMock) -> None:
"""Test the partition key does not log 'Could not determine type'."""
self.write_files({"visit/yr=2023/p1.parquet": {"visit_id": [1]}})
- get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
+ assert orm is not None
+ add_guessed_keys(orm, {"visit"})
mock_logger.warning.assert_not_called()
def test_same_part_names_in_different_tables_do_not_collide(self) -> None:
@@ -200,7 +205,9 @@ def test_same_part_names_in_different_tables_do_not_collide(self) -> None:
"drug/yr=2023/p00001.parquet": {"drug_id": [1], "dose": [1.5]},
}
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
+ assert orm is not None
+ add_guessed_keys(orm, {"drug", "visit"})
assert orm is not None
self.assertSetEqual(set(orm.keys()), {"visit", "drug"})
self.assertSetEqual(
@@ -208,7 +215,8 @@ def test_same_part_names_in_different_tables_do_not_collide(self) -> None:
)
self.assertSetEqual(set(orm["visit"]["columns"].keys()), {"visit_id", "yr"})
- def test_files_and_directories_mix(self) -> None:
+ @patch("datafaker.parquet2orm.logger")
+ def test_files_and_directories_mix(self, mock_logger: MagicMock) -> None:
"""Test a mixture of loose parquet files and parquet dirs are read as separate tables.
Checks empty directory are not treated as partitioned tables."""
self.write_files(
@@ -219,9 +227,12 @@ def test_files_and_directories_mix(self) -> None:
)
(self.parquet_dir / "empty").mkdir()
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
- self.assertSetEqual(set(orm.keys()), {"fruit.parquet", "visit"})
+ table_names = {"fruit.parquet", "visit"}
+ add_guessed_keys(orm, table_names)
+ self.assertSetEqual(set(orm.keys()), table_names)
+ mock_logger.warning.assert_called_once_with("Entry %s is empty", "empty")
def test_nested_plain_directories_are_one_table(self) -> None:
"""Test a subdirectory of unpartitioned files is one table."""
@@ -231,34 +242,43 @@ def test_nested_plain_directories_are_one_table(self) -> None:
"sub/b.parquet": {"x": [2]},
}
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, {"sub"})
self.assertSetEqual(set(orm.keys()), {"sub"})
def test_parquet_dir_that_is_a_dataset(self) -> None:
"""Test the directory given can itself be a partitioned table."""
self.write_files(
{
- "yr=2022/p1.parquet": {"visit_id": [1]},
- "yr=2023/p1.parquet": {"visit_id": [2]},
+ "visit/yr=2022/p1.parquet": {"visit_id": [1]},
+ "visit/yr=2023/p1.parquet": {"visit_id": [2]},
}
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
- self.assertSetEqual(set(orm.keys()), {self.parquet_dir.resolve().name})
+ add_guessed_keys(orm, {"visit"})
+ self.assertSetEqual(set(orm.keys()), {"visit"})
only = next(iter(orm.values()))
self.assertSetEqual(set(only["columns"].keys()), {"visit_id", "yr"})
- def test_parquet_dir_dataset_must_be_consistent(self) -> None:
+ @patch("datafaker.parquet2orm.logger")
+ def test_parquet_dir_dataset_must_be_consistent(
+ self, mock_logger: MagicMock
+ ) -> None:
"""Test a directory of unrelated hive-named tables is still rejected."""
self.write_files(
{
- "yr=2022/p1.parquet": {"visit_id": [1]},
- "yr=2023/p1.parquet": {"other": ["x"]},
+ "visit/yr=2022/p1.parquet": {"visit_id": [1]},
+ "visit/yr=2023/p1.parquet": {"other": ["x"]},
}
)
- with self.assertRaises(Exit):
- get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
+ assert orm is not None
+ self.assertDictEqual(orm, {})
+ mock_logger.error.assert_called_once_with(
+ "Entry %s cannot be read as parquet", "visit"
+ )
@patch("datafaker.parquet2orm.logger")
def test_differing_columns_are_rejected(self, mock_logger: MagicMock) -> None:
@@ -269,20 +289,16 @@ def test_differing_columns_are_rejected(self, mock_logger: MagicMock) -> None:
"sub/b.parquet": {"drug_id": [1]},
}
)
- with self.assertRaises(Exit):
- get_parquet_orm(self.parquet_dir)
- mock_logger.error.assert_called_once()
- message = (
- mock_logger.error.call_args.args[0] % mock_logger.error.call_args.args[1:]
+ orm = get_initial_parquet_orm(self.parquet_dir)
+ assert orm is not None
+ self.assertDictEqual(orm, {})
+ mock_logger.error.assert_called_once_with(
+ "Entry %s cannot be read as parquet", "sub"
)
- self.assertIn("sub", message)
- self.assertIn("missing columns ['visit_id', 'ward']", message)
- self.assertIn("extra columns ['drug_id']", message)
- self.assertIn("move each table into its own directory", message)
@patch("datafaker.parquet2orm.logger")
- def test_differing_types_are_a_warning(self, mock_logger: MagicMock) -> None:
- """Test the same column with a different type warns but is accepted."""
+ def test_differing_types_is_disallowed(self, mock_logger: MagicMock) -> None:
+ """Test the same column with a different type causes an error."""
self.write_files(
{
"visit/yr=2022/p1.parquet": {"visit_id": [1]},
@@ -290,14 +306,12 @@ def test_differing_types_are_a_warning(self, mock_logger: MagicMock) -> None:
"visit/yr=2023/p2.parquet": {"visit_id": [2.5]},
}
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
- self.assertEqual(orm["visit"]["columns"]["visit_id"]["type"], "INTEGER")
- mock_logger.error.assert_not_called()
- mock_logger.warning.assert_called_once()
- args = mock_logger.warning.call_args.args
- self.assertEqual(args[1:3], ("visit", "visit_id"))
- self.assertEqual(args[3], ["float64", "int64"])
+ self.assertDictEqual(orm, {})
+ mock_logger.error.assert_called_once_with(
+ "Entry %s cannot be read as parquet", "visit"
+ )
@patch("datafaker.parquet2orm.logger")
def test_int32_and_nullable_int32_are_the_same_type(
@@ -316,11 +330,14 @@ def test_int32_and_nullable_int32_are_the_same_type(
pd.DataFrame({"age": pd.array([None, 2], dtype="Int32")}).to_parquet(
self.parquet_dir / "person/yr=2023/p1.parquet"
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, {"person"})
self.assertEqual(orm["person"]["columns"]["age"]["type"], "INTEGER")
- for call in mock_logger.warning.call_args_list:
- self.assertNotIn("different types", call.args[0])
+ mock_logger.warning.assert_called_once_with(
+ "Found no likely primary keys for table %s",
+ "person",
+ )
def test_nullable_int_first_file_is_still_integer(self) -> None:
"""Test a pandas ``Int32`` column is typed ``INTEGER``."""
@@ -328,23 +345,13 @@ def test_nullable_int_first_file_is_still_integer(self) -> None:
pd.DataFrame({"age": pd.array([None, 2], dtype="Int32")}).to_parquet(
self.parquet_dir / "person/yr=2022/p1.parquet"
)
- orm = get_parquet_orm(self.parquet_dir)
+ orm = get_initial_parquet_orm(self.parquet_dir)
assert orm is not None
+ add_guessed_keys(orm, {"person"})
self.assertEqual(orm["person"]["columns"]["age"]["type"], "INTEGER")
- def test_differing_partition_keys_are_rejected(self) -> None:
- """Test files partitioned differently are rejected."""
- self.write_files(
- {
- "visit/yr=2022/p1.parquet": {"visit_id": [1]},
- "visit/p2.parquet": {"visit_id": [2]},
- }
- )
- with self.assertRaises(Exit):
- get_parquet_orm(self.parquet_dir)
-
-class PartitionedParquetEngine(TestCase):
+class PartitionedParquetEngine(DatafakerTestCase):
"""Integration tests for querying Parquet data through the parquet_dir option."""
def setUp(self) -> None:
diff --git a/tests/test_utils_mssql.py b/tests/test_utils_mssql.py
index 5e27cd8f..44748ed0 100644
--- a/tests/test_utils_mssql.py
+++ b/tests/test_utils_mssql.py
@@ -107,8 +107,9 @@ def test_duckdb_parquet_dir_sets_search_path(self) -> None:
if len(calls[0].args) > 1
else calls[0].kwargs.get("settings", {})
)
- self.assertIn("file_search_path", settings_passed)
- self.assertEqual(settings_passed["file_search_path"], f"'{parq_dir}'")
+ all_settings = "\n".join(settings_passed)
+ self.assertIn("file_search_path", all_settings)
+ self.assertIn(f"SET file_search_path TO '{parq_dir}';", all_settings)
def test_mssql_dsn_schema_sets_translate_map(self) -> None:
"""schema_translate_map is set even for an MS-SQL DSN (engine creation, no connect)."""
@@ -144,9 +145,11 @@ def test_reflect_called_with_schema(self) -> None:
mock_meta_data.return_value = mock_md
mock_md.reflect.return_value = None
- get_metadata(mock_engine, schema_name="myschema")
+ get_metadata(mock_engine, schema_name="myschema", views_to_tables=False)
- mock_md.reflect.assert_called_once_with(mock_engine, schema="myschema")
+ mock_md.reflect.assert_called_once_with(
+ mock_engine, schema="myschema", views=False
+ )
def test_reflect_called_without_schema_when_none(self) -> None:
"""get_metadata passes schema=None to reflect when no schema_name is given."""
@@ -158,6 +161,8 @@ def test_reflect_called_without_schema_when_none(self) -> None:
mock_meta_data.return_value = mock_md
mock_md.reflect.return_value = None
- get_metadata(mock_engine)
+ get_metadata(mock_engine, schema_name=None, views_to_tables=False)
- mock_md.reflect.assert_called_once_with(mock_engine, schema=None)
+ mock_md.reflect.assert_called_once_with(
+ mock_engine, schema=None, views=False
+ )
diff --git a/tests/utils.py b/tests/utils.py
index 8bdf5057..6673d7c5 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -189,7 +189,7 @@ class DuckTestDb(TestDatabaseBase):
SQL_REMOVALS_RE = re.compile(
r"^(CREATE DATABASE .*;)|(\\.*)|(ALTER [^;]*;)|(\bUSING\s+btree)", re.MULTILINE
)
- # Postgres' default stupid time format, ingoring time zone for now
+ # Postgres' default stupid time format, ignoring time zone for now
TIME_FORMAT_RE = re.compile(
r"'([A-Z][a-z]+ \d+ \d\d:\d\d:\d\d \d\d\d\d) ([A-Z+\-0-9:]+)'"
)
@@ -284,7 +284,7 @@ class MsSqlTestDb(TestDatabaseBase):
_EOL_RE = re.compile(r";$", re.MULTILINE)
_TIMESTAMP_RE = re.compile(r"(TIMESTAMP)\s+WITH\s+TIME\s+ZONE", re.IGNORECASE)
_ALTER_OWNER_RE = re.compile(
- r'ALTER\s+(TABLE|DATABASE)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;'
+ r'ALTER\s+(TABLE|DATABASE|VIEW)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;'
)
_ALTER_ONLY_RE = re.compile(r"ALTER\s+TABLE\s+ONLY\s+")
_CREATE_INDEX_RE = re.compile(
@@ -764,7 +764,7 @@ def setUp(self) -> None:
# Generate the `orm.yaml` from the database
(self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True)
with os.fdopen(self.orm_fd, "w", encoding="utf-8") as orm_fh:
- orm_fh.write(make_tables_file(self.dsn, self.schema_name))
+ orm_fh.write(make_tables_file(self.dsn, self.schema_name, False))
# Create a separate empty destination database
self.make_destination_database("dst")