Skip to content
Merged
Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
2e1daa8
feat: support timestamp_precision in table schema
Linchin Nov 20, 2025
253ac1f
undelete test_to_api_repr_with_subfield
Linchin Nov 20, 2025
dc3c498
lint
Linchin Nov 20, 2025
97f0251
Merge branch 'main' into pico-sql
Linchin Nov 24, 2025
234a3fd
remove property setter as it is read only
Linchin Nov 24, 2025
b20159b
docstring
Linchin Nov 24, 2025
a1bc2cb
update unit test
Linchin Nov 24, 2025
bc6dcda
unit test
Linchin Nov 24, 2025
518a12c
add create_table system test
Linchin Nov 24, 2025
a8d5f5c
typo
Linchin Nov 24, 2025
0567adf
add query system test
Linchin Nov 25, 2025
1268c45
remove query system test
Linchin Nov 25, 2025
8603973
unit test
Linchin Nov 25, 2025
cb9f818
unit test
Linchin Nov 25, 2025
696dfff
unit test
Linchin Nov 25, 2025
d24df7d
docstring
Linchin Nov 25, 2025
873bff6
docstring
Linchin Nov 25, 2025
6a93c26
docstring
Linchin Nov 25, 2025
9a4f72f
improve __repr__()
Linchin Nov 25, 2025
c146e39
Merge branch 'main' into pico-sql
Linchin Nov 26, 2025
fc08533
use enum for timestamp_precision
Linchin Dec 8, 2025
0b743f3
delete file
Linchin Dec 8, 2025
2a81ef9
docstring
Linchin Dec 8, 2025
e131b6d
update test
Linchin Dec 8, 2025
7693537
improve unit test
Linchin Dec 8, 2025
5d2fbf0
fix system test
Linchin Dec 8, 2025
c7c2b47
Update tests/system/test_client.py
Linchin Dec 9, 2025
f87b618
only allow enums values
Linchin Dec 11, 2025
c0e4595
docstring and tests
Linchin Dec 11, 2025
04c5f59
handle server inconsistency and unit tests
Linchin Dec 11, 2025
255b87a
Merge branch 'main' into pico-sql
Linchin Dec 17, 2025
657dd84
Merge branch 'main' into pico-sql
Linchin Dec 17, 2025
4cd3df4
improve code
Linchin Dec 19, 2025
e6a3f8b
docstring
Linchin Dec 19, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions google/cloud/bigquery/enums.py
Original file line number Diff line number Diff line change
Expand Up @@ -480,3 +480,18 @@ class SourceColumnMatch(str, enum.Enum):
NAME = "NAME"
"""Matches by name. This reads the header row as column names and reorders
columns to match the field names in the schema."""


class TimestampPrecision(object):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at the code again after our talk, I still think this should be turned into a true enum.

With the current code, TimestampPrecision is just a shortcut to creating int or None values. The argument type can't be TimeStampPrecision. This means all the docstrings have to describe all the values as Union[int | None], and then do more work to describe how to build one or what those special values mean, and we lose a lot of the value of encapsulating it in an enum to begin with

If we change this to actually be an enum, we will have to convert it back to it's raw value before sending it to the backend. But it should make typing a lot cleaner

Copy link
Contributor

@daniel-sanche daniel-sanche Dec 9, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But if you want to leave it as-is, that's fine. We'd just need to update the typing and docstrings to be accurate

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

As we have discussed offline, I have updated to use enum.Enum, and took the liberty to support None to ensure backward compatibility regarding string representation

"""Precision (maximum number of total digits in base 10) for seconds of
TIMESTAMP type."""

MICROSECOND = None
"""
Default, for TIMESTAMP type with microsecond precision.
"""

PICOSECOND = 12
"""
For TIMESTAMP type with picosecond precision.
"""
24 changes: 21 additions & 3 deletions google/cloud/bigquery/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,14 @@ class SchemaField(object):

Only valid for top-level schema fields (not nested fields).
If the type is FOREIGN, this field is required.

timestamp_precision: Union[enums.TimestampPrecision, int, None]
Precision (maximum number of total digits in base 10) for seconds
of TIMESTAMP type.

Defaults to `enums.TimestampPrecision.MICROSECOND` (`None`) for
microsecond precision. Use `enums.TimestampPrecision.PICOSECOND`
(`12`) for picosecond precision.
"""

def __init__(
Expand All @@ -213,6 +221,7 @@ def __init__(
range_element_type: Union[FieldElementType, str, None] = None,
rounding_mode: Union[enums.RoundingMode, str, None] = None,
foreign_type_definition: Optional[str] = None,
timestamp_precision: Union[enums.TimestampPrecision, int, None] = None,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This type annotation isn't actually right with the current code, since it won't ever be a TimestampPrecision instance

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This annotation is for the type accepted, so I think it is accurate?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated to reflect the new types accepted: None and enum. The reason why None is accepted is to ensure an empty api representation when there is no user input, so the string representation remains the same if timestamp_precision is not set

):
self._properties: Dict[str, Any] = {
"name": name,
Expand All @@ -237,6 +246,8 @@ def __init__(
if isinstance(policy_tags, PolicyTagList)
else None
)
if timestamp_precision is not None:
self._properties["timestampPrecision"] = timestamp_precision
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you make TimestampPrecision into an enum, you can save this as a primitive here, and then rebuild in the property

self._properties["timestampPrecision"] = timestamp_precision.value if isinstance(TimestampPrecision) else timestamp_precision

if isinstance(range_element_type, str):
self._properties["rangeElementType"] = {"type": range_element_type}
if isinstance(range_element_type, FieldElementType):
Expand Down Expand Up @@ -374,6 +385,13 @@ def policy_tags(self):
resource = self._properties.get("policyTags")
return PolicyTagList.from_api_repr(resource) if resource is not None else None

@property
def timestamp_precision(self):
"""Union[enums.TimestampPrecision, int, None]: Precision (maximum number
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Like I mentioned in a different comment, we should make sure it's stored in a single format, to make things easier when we go back to read it. I'd recommend making this just return the enum type

Copy link
Contributor Author

@Linchin Linchin Dec 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done, when accessing the property, the user will always get an enum, while the api representation is int or None. We also limited the acceptable inputs to enum or None.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, this doctring doesn't seem to match the format of the others. Shouldn't it be in a returns block? And I'd suggest adding the type as an annotation, so it can be caught by mypy

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

both docstring and type annotation done

of total digits in base 10) for seconds of TIMESTAMP type.
"""
return _helpers._int_or_none(self._properties.get("timestampPrecision"))

def to_api_repr(self) -> dict:
"""Return a dictionary representing this schema field.

Expand Down Expand Up @@ -417,6 +435,7 @@ def _key(self):
self.description,
self.fields,
policy_tags,
self.timestamp_precision,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if you decide to do an Enum subclass, this could look like: self.timestamp_precision.value

)

def to_standard_sql(self) -> standard_sql.StandardSqlField:
Expand Down Expand Up @@ -467,10 +486,9 @@ def __hash__(self):
return hash(self._key())

def __repr__(self):
key = self._key()
policy_tags = key[-1]
*initial_tags, policy_tags, timestamp_precision_tag = self._key()
policy_tags_inst = None if policy_tags is None else PolicyTagList(policy_tags)
adjusted_key = key[:-1] + (policy_tags_inst,)
adjusted_key = (*initial_tags, policy_tags_inst, timestamp_precision_tag)
return f"{self.__class__.__name__}{adjusted_key}"


Expand Down
23 changes: 23 additions & 0 deletions tests/system/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,16 @@
bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
]
SCHEMA_PICOSECOND = [
bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
bigquery.SchemaField(
"time_pico",
"TIMESTAMP",
mode="REQUIRED",
timestamp_precision=enums.TimestampPrecision.PICOSECOND,
),
]
CLUSTERING_SCHEMA = [
bigquery.SchemaField("full_name", "STRING", mode="REQUIRED"),
bigquery.SchemaField("age", "INTEGER", mode="REQUIRED"),
Expand Down Expand Up @@ -631,6 +641,19 @@ def test_create_table_w_time_partitioning_w_clustering_fields(self):
self.assertEqual(time_partitioning.field, "transaction_time")
self.assertEqual(table.clustering_fields, ["user_email", "store_code"])

def test_create_tabl_w_picosecond_timestamp(self):
dataset = self.temp_dataset(_make_dataset_id("create_table"))
table_id = "test_table"
table_arg = Table(dataset.table(table_id), schema=SCHEMA_PICOSECOND)
self.assertFalse(_table_exists(table_arg))

table = helpers.retry_403(Config.CLIENT.create_table)(table_arg)
self.to_delete.insert(0, table)

self.assertTrue(_table_exists(table))
self.assertEqual(table.table_id, table_id)
self.assertEqual(table.schema, SCHEMA_PICOSECOND)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we have a test that reads back a timestamp, and makes sure its in the expected range? Or am I misunderstanding?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR only involves creating and reading table schema that has picosecond timestamp. I think we can add the tests in the PR supporting writing to and reading from the table.


def test_delete_dataset_with_string(self):
dataset_id = _make_dataset_id("delete_table_true_with_string")
project = Config.CLIENT.project
Expand Down
37 changes: 36 additions & 1 deletion tests/unit/test_schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def test_constructor_explicit(self):
default_value_expression=FIELD_DEFAULT_VALUE_EXPRESSION,
rounding_mode=enums.RoundingMode.ROUNDING_MODE_UNSPECIFIED,
foreign_type_definition="INTEGER",
timestamp_precision=enums.TimestampPrecision.PICOSECOND,
)
self.assertEqual(field.name, "test")
self.assertEqual(field.field_type, "STRING")
Expand All @@ -87,6 +88,10 @@ def test_constructor_explicit(self):
)
self.assertEqual(field.rounding_mode, "ROUNDING_MODE_UNSPECIFIED")
self.assertEqual(field.foreign_type_definition, "INTEGER")
self.assertEqual(
field._properties["timestampPrecision"],
enums.TimestampPrecision.PICOSECOND,
)

def test_constructor_explicit_none(self):
field = self._make_one("test", "STRING", description=None, policy_tags=None)
Expand Down Expand Up @@ -189,6 +194,23 @@ def test_to_api_repr_with_subfield(self):
},
)

def test_to_api_repr_w_timestamp_precision(self):
field = self._make_one(
"foo",
"TIMESTAMP",
"NULLABLE",
timestamp_precision=enums.TimestampPrecision.PICOSECOND,
)
self.assertEqual(
field.to_api_repr(),
{
"mode": "NULLABLE",
"name": "foo",
"type": "TIMESTAMP",
"timestampPrecision": enums.TimestampPrecision.PICOSECOND,
},
)

def test_from_api_repr(self):
field = self._get_target_class().from_api_repr(
{
Expand All @@ -198,6 +220,7 @@ def test_from_api_repr(self):
"name": "foo",
"type": "record",
"roundingMode": "ROUNDING_MODE_UNSPECIFIED",
"timestampPrecision": enums.TimestampPrecision.PICOSECOND,
}
)
self.assertEqual(field.name, "foo")
Expand All @@ -210,6 +233,10 @@ def test_from_api_repr(self):
self.assertEqual(field.fields[0].mode, "NULLABLE")
self.assertEqual(field.range_element_type, None)
self.assertEqual(field.rounding_mode, "ROUNDING_MODE_UNSPECIFIED")
self.assertEqual(
field._properties["timestampPrecision"],
enums.TimestampPrecision.PICOSECOND,
)

def test_from_api_repr_policy(self):
field = self._get_target_class().from_api_repr(
Expand Down Expand Up @@ -323,6 +350,12 @@ def test_foreign_type_definition_property_str(self):
schema_field._properties["foreignTypeDefinition"] = FOREIGN_TYPE_DEFINITION
self.assertEqual(schema_field.foreign_type_definition, FOREIGN_TYPE_DEFINITION)

def test_timestamp_precision_property(self):
TIMESTAMP_PRECISION = enums.TimestampPrecision.PICOSECOND
schema_field = self._make_one("test", "TIMESTAMP")
schema_field._properties["timestampPrecision"] = TIMESTAMP_PRECISION
self.assertEqual(schema_field.timestamp_precision, TIMESTAMP_PRECISION)

def test_to_standard_sql_simple_type(self):
examples = (
# a few legacy types
Expand Down Expand Up @@ -637,7 +670,9 @@ def test___hash__not_equals(self):

def test___repr__(self):
field1 = self._make_one("field1", "STRING")
expected = "SchemaField('field1', 'STRING', 'NULLABLE', None, None, (), None)"
expected = (
"SchemaField('field1', 'STRING', 'NULLABLE', None, None, (), None, None)"
)
self.assertEqual(repr(field1), expected)

def test___repr__evaluable_no_policy_tags(self):
Expand Down