diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0fdff62..995eb4b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -42,7 +42,6 @@ jobs: run: python3 -m pip install -e ".[dev]" - name: Run tests and collect coverage run: | - mv credentials.json.sample credentials.json pytest --cov snowexsql --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d661011..ee2fef2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -54,5 +54,4 @@ jobs: python3 -m pip install -e ".[dev]" - name: Test with pytest run: | - mv credentials.json.sample credentials.json pytest -s diff --git a/.gitignore b/.gitignore index a801518..b03c83f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,7 +17,7 @@ docs/_build/* **/.ipynb_checkpoints/* # Database credential information -credentials.json +./credentials.json # Test related files .coverage diff --git a/README.rst b/README.rst index 676335c..bbedd1b 100644 --- a/README.rst +++ b/README.rst @@ -38,20 +38,50 @@ Features .. _examples: https://snowexsql.readthedocs.io/en/latest/gallery/index.html -Installing ----------- -If you are just planning on using the database, then only install the -python package instructions below. +Setup +----- -I just want to use it ---------------------- +Installing +========== Install using pip: .. code-block:: pip install snowexsql -I want data fast +Configuring the database connection +=================================== +Using this library requires setting up the database connection credentials. +There are two options to do this: + +* Set database connection URL via ``SNOWEX_DB_CONNECTION`` environment variable + Example: + +.. code-block:: bash + + export SNOWEX_DB_CONNECTION="user:password@127.0.0.1/db_name" + +* Point to a credentials JSON file via ``SNOWEX_DB_CREDENTIALS`` environment variable + Example + +.. code-block:: bash + + export SNOWEX_DB_CREDENTIALS="/path/to/credentials.json" + + +`Sample JSON file <./credentials.json.sample>`_: + +.. code-block:: json + + { + "address": "localhost", + "db_name": "snowexdb", + "username": "user", + "password": "password" + } + + +Accessing the SnowEx data ----------------- A programmatic API has been created for fast and standard access to Point and Layer data. There are two examples_ covering the @@ -69,24 +99,13 @@ detailed description. ) print(df.head()) -I need help + +Getting help ------------ Jump over to `our discussion forum `_ and get help from our community. -I want to contribute ---------------------- -Thank you for the interest! - -Our community follows the |Contributor Covenant| - -.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg - :target: code_of_conduct.md -.. _contribution guide: https://snowexsql.readthedocs.io/en/latest/community/contributing.html - -Have a look at our `contribution guide`_ and see the many ways to get involved! - Documentation ------------- @@ -111,6 +130,42 @@ last image submitted to GitHub. make docs + +I want to contribute +--------------------- +Thank you for the interest! + +Our community follows the |Contributor Covenant| + +.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg + :target: code_of_conduct.md +.. _contribution guide: https://snowexsql.readthedocs.io/en/latest/community/contributing.html + +Have a look at our `contribution guide`_ and see the many ways to get involved! + +Testing +======= +To run the test suite locally requires having a running instance of PostgreSQL. +The test suite is configured to run against these credentials: + +.. code-block:: + + builder:db_builder@localhost/test + +This requires a running database on ``localhost`` where the user ``builder`` has access +to the ``test`` database with the password ``db_builder``. + +It is possible to set a custom host and database via the ``SNOWEX_TEST_DB`` environment +variable. Example that would connect to a server on ``my.host`` and the database +``snowex_test``: + +.. code-block:: bash + + export SNOWEX_TEST_DB="my_host/snowex_test" + +More on connection strings to PostgreSQL can be found with the +`official documentation `_. + DOI --- .. |HW22| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.7618102.svg diff --git a/credentials.json.sample b/credentials.json.sample index 572f9b1..bd31770 100644 --- a/credentials.json.sample +++ b/credentials.json.sample @@ -1,14 +1,6 @@ { - "production": { - "address": "localhost", - "db_name": "snowexsql", - "username": "builder", - "password": "db_builder" - }, - "tests": { - "address": "localhost", - "db_name": "test", - "username": "builder", - "password": "db_builder" - } + "address": "localhost", + "db_name": "test", + "username": "user", + "password": "password" } diff --git a/snowexsql/db.py b/snowexsql/db.py index 9a2c6a8..cf9f56b 100644 --- a/snowexsql/db.py +++ b/snowexsql/db.py @@ -1,21 +1,18 @@ """ -This module contains tool used directly regarding the database. This includes -getting a session, initializing the database, getting table attributes, etc. +This module handles loading the database connection information and creating +a session. """ import json import os from contextlib import contextmanager -from pathlib import Path from snowexsql.tables.base import Base from sqlalchemy import MetaData, create_engine from sqlalchemy.orm import sessionmaker -# The default credentials file name -CREDENTIAL_FILE="credentials.json" # This library requires a postgres dialect and the psycopg2 driver -DB_CONNECTION_PROTOCOL = 'postgresql+psycopg2://' +DB_CONNECTION_PROTOCOL = "postgresql+psycopg2://" # Always create a Session in UTC time DB_CONNECTION_OPTIONS = {"options": "-c timezone=UTC"} @@ -43,19 +40,28 @@ def load_credentials(credentials_path=None): Returns: Dictionary - Credential information """ - if credentials_path is None: - credentials_path = Path(__file__).parent.parent / CREDENTIAL_FILE + credentials = None - with open(credentials_path) as file: - credentials = json.load(file) + if credentials_path is not None: + with open(credentials_path) as file: + credentials = json.load(file) + elif os.getenv("SNOWEX_DB_CREDENTIALS"): + with open(os.getenv("SNOWEX_DB_CREDENTIALS")) as file: + credentials = json.load(file) - if os.getenv('SNOWEXSQL_TESTS', False): - return credentials['tests'] - else: - return credentials['production'] + if credentials is None: + raise FileNotFoundError( + "File to credentials was not provided. Please use the following options: " + "* Pass the file path to this method " + "* Set the SNOWEX_DB_CREDENTIALS environment variable to a JSON file" + "* Set the SNOWEX_DB_CONNECTION environment variable. " + " Example: user:password@127.0.0.1/db_name" + ) + return credentials -def db_connection_string(credentials_path=None): + +def db_connection_string(credentials_path:str = None): """ Construct a connection info string for SQLAlchemy database @@ -65,16 +71,22 @@ def db_connection_string(credentials_path=None): Returns: String - DB connection """ - credentials = load_credentials(credentials_path) - db = DB_CONNECTION_PROTOCOL - db += f"{credentials['username']}:{credentials['password']}"\ - f"@{credentials['address']}/{credentials['db_name']}" + + if os.getenv("SNOWEX_DB_CONNECTION"): + credentials = os.getenv("SNOWEX_DB_CONNECTION") + db += credentials + else: + credentials = load_credentials(credentials_path) + db += ( + f"{credentials['username']}:{credentials['password']}" + f"@{credentials['address']}/{credentials['db_name']}" + ) return db -def get_db(credentials_path=None, return_metadata=False): +def get_db(credentials_path: str = None, return_metadata: bool = False): """ Returns the DB engine, MetaData, and session object @@ -126,8 +138,8 @@ def get_table_attributes(DataCls): Returns a list of all the table columns to be used for each entry """ - valid_attributes = [att for att in dir(DataCls) if att[0] != '_'] + valid_attributes = [att for att in dir(DataCls) if att[0] != "_"] # Drop ID as it is (should) never provided - valid_attributes = [v for v in valid_attributes if v != 'id'] + valid_attributes = [v for v in valid_attributes if v != "id"] return valid_attributes diff --git a/tests/conftest.py b/tests/conftest.py index a694c1f..ae06ce2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,22 +1,33 @@ import os from contextlib import contextmanager +from pathlib import Path import pytest import snowexsql from pytest_factoryboy import register -from snowexsql.db import ( - DB_CONNECTION_OPTIONS, db_connection_string, initialize -) +from snowexsql.db import DB_CONNECTION_OPTIONS, db_connection_string, initialize from sqlalchemy import create_engine +from tests.factories import ( + CampaignFactory, + DOIFactory, + InstrumentFactory, + LayerDataFactory, + LayerDensityFactory, + LayerTemperatureFactory, + MeasurementTypeFactory, + ObserverFactory, + PointDataFactory, + PointObservationFactory, + SiteFactory, +) + from tests import SESSION -from tests.factories import (CampaignFactory, DOIFactory, InstrumentFactory, - LayerDataFactory, LayerDensityFactory, - LayerTemperatureFactory, MeasurementTypeFactory, - ObserverFactory, PointDataFactory, - PointObservationFactory, SiteFactory) -# Environment variable to load the correct credentials -os.environ['SNOWEXSQL_TESTS'] = 'True' +# Environment variable to load the custom DB test credentials +if os.getenv("SNOWEX_TEST_DB") is None: + os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@localhost/test" +else: + os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@" + os.getenv("SNOWEX_TEST_DB") # Make factories available to tests register(CampaignFactory) @@ -35,6 +46,12 @@ def session_maker(): return + +@pytest.fixture(scope="session") +def data_dir(): + return Path(__file__).parent.joinpath("data").absolute().resolve() + + # Add this factory to a test if you would like to debug the SQL statement # It will print the query from the BaseDataset.from_filter() method @pytest.fixture(scope='session') diff --git a/tests/data/credentials.json b/tests/data/credentials.json new file mode 100644 index 0000000..12b59f3 --- /dev/null +++ b/tests/data/credentials.json @@ -0,0 +1,6 @@ +{ + "address": "localhost", + "db_name": "test", + "username": "builder", + "password": "db_builder" +} diff --git a/tests/test_db.py b/tests/test_db.py index ce9d2d3..04e3e04 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -2,74 +2,95 @@ import pytest import snowexsql -from snowexsql.db import (DB_CONNECTION_PROTOCOL, db_connection_string, - db_session_with_credentials, get_db, - load_credentials) -from sqlalchemy import Engine, MetaData +from snowexsql.db import ( + DB_CONNECTION_PROTOCOL, + db_connection_string, + db_session_with_credentials, + get_db, + load_credentials, +) +from sqlalchemy import Engine, MetaData, text from sqlalchemy.orm import Session -from sqlalchemy import text -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def db_connection_string_patch(monkeypatch, test_db_info): def db_string(_credentials): return test_db_info - monkeypatch.setattr( - snowexsql.db, - 'db_connection_string', - db_string - ) + monkeypatch.setattr(snowexsql.db, "db_connection_string", db_string) + + +@pytest.fixture(scope="function") +def db_credentials_file(data_dir): + # Save the connection info for the entire test suite and restore after + default_connection = os.environ.get("SNOWEX_DB_CONNECTION") + del os.environ["SNOWEX_DB_CONNECTION"] + + os.environ["SNOWEX_DB_CREDENTIALS"] = str(data_dir / "credentials.json") + yield + del os.environ["SNOWEX_DB_CREDENTIALS"] + + os.environ["SNOWEX_DB_CONNECTION"] = default_connection class TestDBConnectionInfo: - def test_load_credentials(self): + def test_load_credentials_from_file(self, data_dir): + credentials = load_credentials(data_dir / "credentials.json") + + assert len(credentials.keys()) == 4 + assert "address" in credentials + assert "db_name" in credentials + assert "username" in credentials + assert "password" in credentials + + @pytest.mark.usefixtures("db_credentials_file") + def test_load_credentials_from_file_via_env(self): credentials = load_credentials() assert len(credentials.keys()) == 4 - assert 'address' in credentials - assert 'db_name' in credentials - assert 'username' in credentials - assert 'password' in credentials + assert "address" in credentials + assert "db_name" in credentials + assert "username" in credentials + assert "password" in credentials + + def test_load_credentials_file_not_set(self): + with pytest.raises(FileNotFoundError): + load_credentials() + + def test_load_credentials_file_missing(self): + with pytest.raises(FileNotFoundError): + load_credentials("/tmp/fake_file.json") - def test_db_connection_string_info(self): + def test_db_connection_string_info_from_env(self): + default_connection = os.environ.get("SNOWEX_DB_CONNECTION") db_string = db_connection_string() - credentials = load_credentials() assert db_string.startswith(DB_CONNECTION_PROTOCOL) - assert f"{credentials['username']}:{credentials['password']}" in db_string - assert f"{credentials['address']}/{credentials['db_name']}" in db_string - - def test_load_credentials_production(self): - # Test that a missing environ key will not cause the lookup to fail - del os.environ['SNOWEXSQL_TESTS'] + assert db_string.endswith(default_connection) + @pytest.mark.usefixtures("db_credentials_file") + def test_db_connection_string_info_from_file(self): + db_string = db_connection_string() credentials = load_credentials() - assert len(credentials.keys()) == 4 - assert 'address' in credentials - assert 'db_name' in credentials - assert 'username' in credentials - assert 'password' in credentials - # Set again for subsequent tests - os.environ['SNOWEXSQL_TESTS'] = 'True' + assert db_string.startswith(DB_CONNECTION_PROTOCOL) + assert f"{credentials['username']}:{credentials['password']}" in db_string + assert f"{credentials['address']}/{credentials['db_name']}" in db_string - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_returns_engine(self, monkeypatch, test_db_info): assert isinstance(get_db()[0], Engine) - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_returns_session(self): assert isinstance(get_db()[1], Session) - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_returns_metadata(self): - assert isinstance( - get_db(return_metadata=True)[2], - MetaData - ) + assert isinstance(get_db(return_metadata=True)[2], MetaData) - @pytest.mark.usefixtures('db_connection_string_patch') + @pytest.mark.usefixtures("db_connection_string_patch") def test_db_session_with_credentials(self, monkeypatch, test_db_info): engine, session = None, None with db_session_with_credentials() as (test_engine, test_session): @@ -79,15 +100,13 @@ def test_db_session_with_credentials(self, monkeypatch, test_db_info): assert isinstance(engine, Engine) assert isinstance(session, Session) # Query to create a transaction - session.query(text('1')).all() + session.query(text("1")).all() # On session.close(), all transactions should be gone assert session._transaction is None - @pytest.mark.usefixtures('db_connection_string_patch') - @pytest.mark.parametrize("return_metadata, expected_objs", [ - (False, 2), - (True, 3)]) + @pytest.mark.usefixtures("db_connection_string_patch") + @pytest.mark.parametrize("return_metadata, expected_objs", [(False, 2), (True, 3)]) def test_get_metadata(self, return_metadata, expected_objs): """ Test we can receive a connection and opt out of getting the metadata