From bac1b5a45378845f7d6dc35d88e47d5fb58b5066 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Wed, 19 Nov 2025 08:49:43 -0700 Subject: [PATCH 1/3] README - Reorganize sequence and make shorter headings. --- README.rst | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/README.rst b/README.rst index 676335c..56df750 100644 --- a/README.rst +++ b/README.rst @@ -40,18 +40,14 @@ Features Installing ---------- -If you are just planning on using the database, then only install the -python package instructions below. - -I just want to use it ---------------------- Install using pip: .. code-block:: pip install snowexsql -I want data fast + +Accessing the SnowEx data ----------------- A programmatic API has been created for fast and standard access to Point and Layer data. There are two examples_ covering the @@ -69,24 +65,13 @@ detailed description. ) print(df.head()) -I need help + +Getting help ------------ Jump over to `our discussion forum `_ and get help from our community. -I want to contribute ---------------------- -Thank you for the interest! - -Our community follows the |Contributor Covenant| - -.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg - :target: code_of_conduct.md -.. _contribution guide: https://snowexsql.readthedocs.io/en/latest/community/contributing.html - -Have a look at our `contribution guide`_ and see the many ways to get involved! - Documentation ------------- @@ -111,6 +96,20 @@ last image submitted to GitHub. make docs + +I want to contribute +--------------------- +Thank you for the interest! + +Our community follows the |Contributor Covenant| + +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg + :target: code_of_conduct.md +.. _contribution guide: https://snowexsql.readthedocs.io/en/latest/community/contributing.html + +Have a look at our `contribution guide`_ and see the many ways to get involved! + + DOI --- .. |HW22| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.7618102.svg From 47b14bdd5d0aa2ffc68110d6916ff5395835a7c2 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Wed, 19 Nov 2025 09:06:28 -0700 Subject: [PATCH 2/3] DB Credentials - Change architecture to use environment variables. Moving away from hardcoding a path to find the DB credentials file and use environment variables instead. This now support two ways - one is setting the path to the file and the other is setting the database URL directly. This should make it easier to deploy the library and not relying on the file being in the correct location. --- .github/workflows/ci.yaml | 1 - .github/workflows/main.yml | 1 - .gitignore | 2 +- README.rst | 31 +++++++++++ credentials.json.sample | 16 ++---- snowexsql/db.py | 56 +++++++++++-------- tests/conftest.py | 35 ++++++++---- tests/data/credentials.json | 6 ++ tests/test_db.py | 107 +++++++++++++++++++++--------------- 9 files changed, 164 insertions(+), 91 deletions(-) create mode 100644 tests/data/credentials.json diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0fdff62..995eb4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,6 @@ jobs: run: python3 -m pip install -e ".[dev]" - name: Run tests and collect coverage run: | - mv credentials.json.sample credentials.json pytest --cov snowexsql --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d661011..ee2fef2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,5 +54,4 @@ jobs: python3 -m pip install -e ".[dev]" - name: Test with pytest run: | - mv credentials.json.sample credentials.json pytest -s diff --git a/.gitignore b/.gitignore index a801518..b03c83f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ docs/_build/* **/.ipynb_checkpoints/* # Database credential information -credentials.json +./credentials.json # Test related files .coverage diff --git a/README.rst b/README.rst index 56df750..f8c20f3 100644 --- a/README.rst +++ b/README.rst @@ -109,6 +109,37 @@ Our community follows the |Contributor Covenant| Have a look at our `contribution guide`_ and see the many ways to get involved! +Testing +======= +To run the test suite locally requires setting up the database connection credentials. +There are two options to do this: + +* Set database connection URL via ``SNOWEX_DB_CONNECTION`` environment variable + Example: + +.. code-block:: bash + + export SNOWEX_DB_CONNECTION="user:password@127.0.0.1/db_name" + +* Point to a credentials JSON file via ``SNOWEX_DB_CREDENTIALS`` environment variable + Example + +.. code-block:: bash + + export SNOWEX_DB_CREDENTIALS="/path/to/credentials.json" + + +`Sample JSON file <./credentials.json.sample>`_: + +.. code-block:: json + + { + "address": "localhost", + "db_name": "test", + "username": "user", + "password": "password" + } + DOI --- diff --git a/credentials.json.sample b/credentials.json.sample index 572f9b1..bd31770 100644 --- a/credentials.json.sample +++ b/credentials.json.sample @@ -1,14 +1,6 @@ { - "production": { - "address": "localhost", - "db_name": "snowexsql", - "username": "builder", - "password": "db_builder" - }, - "tests": { - "address": "localhost", - "db_name": "test", - "username": "builder", - "password": "db_builder" - } + "address": "localhost", + "db_name": "test", + "username": "user", + "password": "password" } diff --git a/snowexsql/db.py b/snowexsql/db.py index 9a2c6a8..cf9f56b 100644 --- a/snowexsql/db.py +++ b/snowexsql/db.py @@ -1,21 +1,18 @@ """ -This module contains tool used directly regarding the database. This includes -getting a session, initializing the database, getting table attributes, etc. +This module handles loading the database connection information and creating +a session. """ import json import os from contextlib import contextmanager -from pathlib import Path from snowexsql.tables.base import Base from sqlalchemy import MetaData, create_engine from sqlalchemy.orm import sessionmaker -# The default credentials file name -CREDENTIAL_FILE="credentials.json" # This library requires a postgres dialect and the psycopg2 driver -DB_CONNECTION_PROTOCOL = 'postgresql+psycopg2://' +DB_CONNECTION_PROTOCOL = "postgresql+psycopg2://" # Always create a Session in UTC time DB_CONNECTION_OPTIONS = {"options": "-c timezone=UTC"} @@ -43,19 +40,28 @@ def load_credentials(credentials_path=None): Returns: Dictionary - Credential information """ - if credentials_path is None: - credentials_path = Path(__file__).parent.parent / CREDENTIAL_FILE + credentials = None - with open(credentials_path) as file: - credentials = json.load(file) + if credentials_path is not None: + with open(credentials_path) as file: + credentials = json.load(file) + elif os.getenv("SNOWEX_DB_CREDENTIALS"): + with open(os.getenv("SNOWEX_DB_CREDENTIALS")) as file: + credentials = json.load(file) - if os.getenv('SNOWEXSQL_TESTS', False): - return credentials['tests'] - else: - return credentials['production'] + if credentials is None: + raise FileNotFoundError( + "File to credentials was not provided. Please use the following options: " + "* Pass the file path to this method " + "* Set the SNOWEX_DB_CREDENTIALS environment variable to a JSON file" + "* Set the SNOWEX_DB_CONNECTION environment variable. " + " Example: user:password@127.0.0.1/db_name" + ) + return credentials -def db_connection_string(credentials_path=None): + +def db_connection_string(credentials_path:str = None): """ Construct a connection info string for SQLAlchemy database @@ -65,16 +71,22 @@ def db_connection_string(credentials_path=None): Returns: String - DB connection """ - credentials = load_credentials(credentials_path) - db = DB_CONNECTION_PROTOCOL - db += f"{credentials['username']}:{credentials['password']}"\ - f"@{credentials['address']}/{credentials['db_name']}" + + if os.getenv("SNOWEX_DB_CONNECTION"): + credentials = os.getenv("SNOWEX_DB_CONNECTION") + db += credentials + else: + credentials = load_credentials(credentials_path) + db += ( + f"{credentials['username']}:{credentials['password']}" + f"@{credentials['address']}/{credentials['db_name']}" + ) return db -def get_db(credentials_path=None, return_metadata=False): +def get_db(credentials_path: str = None, return_metadata: bool = False): """ Returns the DB engine, MetaData, and session object @@ -126,8 +138,8 @@ def get_table_attributes(DataCls): Returns a list of all the table columns to be used for each entry """ - valid_attributes = [att for att in dir(DataCls) if att[0] != '_'] + valid_attributes = [att for att in dir(DataCls) if att[0] != "_"] # Drop ID as it is (should) never provided - valid_attributes = [v for v in valid_attributes if v != 'id'] + valid_attributes = [v for v in valid_attributes if v != "id"] return valid_attributes diff --git a/tests/conftest.py b/tests/conftest.py index a694c1f..0eba109 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,31 @@ import os from contextlib import contextmanager +from pathlib import Path import pytest import snowexsql from pytest_factoryboy import register -from snowexsql.db import ( - DB_CONNECTION_OPTIONS, db_connection_string, initialize -) +from snowexsql.db import DB_CONNECTION_OPTIONS, db_connection_string, initialize from sqlalchemy import create_engine +from tests.factories import ( + CampaignFactory, + DOIFactory, + InstrumentFactory, + LayerDataFactory, + LayerDensityFactory, + LayerTemperatureFactory, + MeasurementTypeFactory, + ObserverFactory, + PointDataFactory, + PointObservationFactory, + SiteFactory, +) + from tests import SESSION -from tests.factories import (CampaignFactory, DOIFactory, InstrumentFactory, - LayerDataFactory, LayerDensityFactory, - LayerTemperatureFactory, MeasurementTypeFactory, - ObserverFactory, PointDataFactory, - PointObservationFactory, SiteFactory) -# Environment variable to load the correct credentials -os.environ['SNOWEXSQL_TESTS'] = 'True' +# Environment variable to load the DB credentials +if os.getenv('SNOWEX_DB_CONNECTION') is None and os.getenv('SNOWEX_DB_CREDENTIALS') is None: + os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@localhost/test" # Make factories available to tests register(CampaignFactory) @@ -35,6 +44,12 @@ def session_maker(): return + +@pytest.fixture(scope="session") +def data_dir(): + return Path(__file__).parent.joinpath("data").absolute().resolve() + + # Add this factory to a test if you would like to debug the SQL statement # It will print the query from the BaseDataset.from_filter() method @pytest.fixture(scope='session') diff --git a/tests/data/credentials.json b/tests/data/credentials.json new file mode 100644 index 0000000..12b59f3 --- /dev/null +++ b/tests/data/credentials.json @@ -0,0 +1,6 @@ +{ + "address": "localhost", + "db_name": "test", + "username": "builder", + "password": "db_builder" +} diff --git a/tests/test_db.py b/tests/test_db.py index ce9d2d3..04e3e04 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -2,74 +2,95 @@ import pytest import snowexsql -from snowexsql.db import (DB_CONNECTION_PROTOCOL, db_connection_string, - db_session_with_credentials, get_db, - load_credentials) -from sqlalchemy import Engine, MetaData +from snowexsql.db import ( + DB_CONNECTION_PROTOCOL, + db_connection_string, + db_session_with_credentials, + get_db, + load_credentials, +) +from sqlalchemy import Engine, MetaData, text from sqlalchemy.orm import Session -from sqlalchemy import text -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def db_connection_string_patch(monkeypatch, test_db_info): def db_string(_credentials): return test_db_info - monkeypatch.setattr( - snowexsql.db, - 'db_connection_string', - db_string - ) + monkeypatch.setattr(snowexsql.db, "db_connection_string", db_string) + + +@pytest.fixture(scope="function") +def db_credentials_file(data_dir): + # Save the connection info for the entire test suite and restore after + default_connection = os.environ.get("SNOWEX_DB_CONNECTION") + del os.environ["SNOWEX_DB_CONNECTION"] + + os.environ["SNOWEX_DB_CREDENTIALS"] = str(data_dir / "credentials.json") + yield + del os.environ["SNOWEX_DB_CREDENTIALS"] + + os.environ["SNOWEX_DB_CONNECTION"] = default_connection class TestDBConnectionInfo: - def test_load_credentials(self): + def test_load_credentials_from_file(self, data_dir): + credentials = load_credentials(data_dir / "credentials.json") + + assert len(credentials.keys()) == 4 + assert "address" in credentials + assert "db_name" in credentials + assert "username" in credentials + assert "password" in credentials + + @pytest.mark.usefixtures("db_credentials_file") + def test_load_credentials_from_file_via_env(self): credentials = load_credentials() assert len(credentials.keys()) == 4 - assert 'address' in credentials - assert 'db_name' in credentials - assert 'username' in credentials - assert 'password' in credentials + assert "address" in credentials + assert "db_name" in credentials + assert "username" in credentials + assert "password" in credentials + + def test_load_credentials_file_not_set(self): + with pytest.raises(FileNotFoundError): + load_credentials() + + def test_load_credentials_file_missing(self): + with pytest.raises(FileNotFoundError): + load_credentials("/tmp/fake_file.json") - def test_db_connection_string_info(self): + def test_db_connection_string_info_from_env(self): + default_connection = os.environ.get("SNOWEX_DB_CONNECTION") db_string = db_connection_string() - credentials = load_credentials() assert db_string.startswith(DB_CONNECTION_PROTOCOL) - assert f"{credentials['username']}:{credentials['password']}" in db_string - assert f"{credentials['address']}/{credentials['db_name']}" in db_string - - def test_load_credentials_production(self): - # Test that a missing environ key will not cause the lookup to fail - del os.environ['SNOWEXSQL_TESTS'] + assert db_string.endswith(default_connection) + @pytest.mark.usefixtures("db_credentials_file") + def test_db_connection_string_info_from_file(self): + db_string = db_connection_string() credentials = load_credentials() - assert len(credentials.keys()) == 4 - assert 'address' in credentials - assert 'db_name' in credentials - assert 'username' in credentials - assert 'password' in credentials - # Set again for subsequent tests - os.environ['SNOWEXSQL_TESTS'] = 'True' + assert db_string.startswith(DB_CONNECTION_PROTOCOL) + assert f"{credentials['username']}:{credentials['password']}" in db_string + assert f"{credentials['address']}/{credentials['db_name']}" in db_string - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_returns_engine(self, monkeypatch, test_db_info): assert isinstance(get_db()[0], Engine) - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_returns_session(self): assert isinstance(get_db()[1], Session) - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_returns_metadata(self): - assert isinstance( - get_db(return_metadata=True)[2], - MetaData - ) + assert isinstance(get_db(return_metadata=True)[2], MetaData) - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_db_session_with_credentials(self, monkeypatch, test_db_info): engine, session = None, None with db_session_with_credentials() as (test_engine, test_session): @@ -79,15 +100,13 @@ def test_db_session_with_credentials(self, monkeypatch, test_db_info): assert isinstance(engine, Engine) assert isinstance(session, Session) # Query to create a transaction - session.query(text('1')).all() + session.query(text("1")).all() # On session.close(), all transactions should be gone assert session._transaction is None - @pytest.mark.usefixtures('db_connection_string_patch') - @pytest.mark.parametrize("return_metadata, expected_objs", [ - (False, 2), - (True, 3)]) + @pytest.mark.usefixtures("db_connection_string_patch") + @pytest.mark.parametrize("return_metadata, expected_objs", [(False, 2), (True, 3)]) def test_get_metadata(self, return_metadata, expected_objs): """ Test we can receive a connection and opt out of getting the metadata From 1dd2d1ee245b00a12e50c14c922a3843ec9297a4 Mon Sep 17 00:00:00 2001 From: Joachim Meyer Date: Wed, 19 Nov 2025 17:01:55 -0700 Subject: [PATCH 3/3] Tests - Refactor custom DB connection. Use a different variable when a user wants to use different credentials than the default. This only lets the user change the db host and database. The credentials remain the same to reduce the possibility to run the test against a production DB that would wipe out all data. --- README.rst | 69 ++++++++++++++++++++++++++++++++--------------- tests/conftest.py | 6 +++-- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index f8c20f3..bbedd1b 100644 --- a/README.rst +++ b/README.rst @@ -38,14 +38,48 @@ Features .. _examples: https://snowexsql.readthedocs.io/en/latest/gallery/index.html +Setup +----- + Installing ----------- +========== Install using pip: .. code-block:: pip install snowexsql +Configuring the database connection +=================================== +Using this library requires setting up the database connection credentials. +There are two options to do this: + +* Set database connection URL via ``SNOWEX_DB_CONNECTION`` environment variable + Example: + +.. code-block:: bash + + export SNOWEX_DB_CONNECTION="user:password@127.0.0.1/db_name" + +* Point to a credentials JSON file via ``SNOWEX_DB_CREDENTIALS`` environment variable + Example + +.. code-block:: bash + + export SNOWEX_DB_CREDENTIALS="/path/to/credentials.json" + + +`Sample JSON file <./credentials.json.sample>`_: + +.. code-block:: json + + { + "address": "localhost", + "db_name": "snowexdb", + "username": "user", + "password": "password" + } + Accessing the SnowEx data ----------------- @@ -111,35 +145,26 @@ Have a look at our `contribution guide`_ and see the many ways to get involved! Testing ======= -To run the test suite locally requires setting up the database connection credentials. -There are two options to do this: +To run the test suite locally requires having a running instance of PostgreSQL. +The test suite is configured to run against these credentials: -* Set database connection URL via ``SNOWEX_DB_CONNECTION`` environment variable - Example: +.. code-block:: -.. code-block:: bash + builder:db_builder@localhost/test - export SNOWEX_DB_CONNECTION="user:password@127.0.0.1/db_name" +This requires a running database on ``localhost`` where the user ``builder`` has access +to the ``test`` database with the password ``db_builder``. -* Point to a credentials JSON file via ``SNOWEX_DB_CREDENTIALS`` environment variable - Example +It is possible to set a custom host and database via the ``SNOWEX_TEST_DB`` environment +variable. Example that would connect to a server on ``my.host`` and the database +``snowex_test``: .. code-block:: bash - export SNOWEX_DB_CREDENTIALS="/path/to/credentials.json" - - -`Sample JSON file <./credentials.json.sample>`_: - -.. code-block:: json - - { - "address": "localhost", - "db_name": "test", - "username": "user", - "password": "password" - } + export SNOWEX_TEST_DB="my_host/snowex_test" +More on connection strings to PostgreSQL can be found with the +`official documentation `_. DOI --- diff --git a/tests/conftest.py b/tests/conftest.py index 0eba109..ae06ce2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,9 +23,11 @@ from tests import SESSION -# Environment variable to load the DB credentials -if os.getenv('SNOWEX_DB_CONNECTION') is None and os.getenv('SNOWEX_DB_CREDENTIALS') is None: +# Environment variable to load the custom DB test credentials +if os.getenv("SNOWEX_TEST_DB") is None: os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@localhost/test" +else: + os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@" + os.getenv("SNOWEX_TEST_DB") # Make factories available to tests register(CampaignFactory)