Imporove credentials file handling

.github/workflows/ci.yaml

@@ -42,7 +42,6 @@ jobs:
         run: python3 -m pip install -e ".[dev]"
       - name: Run tests and collect coverage
         run: |
-          mv credentials.json.sample credentials.json
           pytest --cov snowexsql --cov-report=xml
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5

.github/workflows/main.yml

@@ -54,5 +54,4 @@ jobs:
           python3 -m pip install -e ".[dev]"
       - name: Test with pytest
         run: |
-          mv credentials.json.sample credentials.json
           pytest -s

.gitignore

@@ -17,7 +17,7 @@ docs/_build/*
 **/.ipynb_checkpoints/*
 
 # Database credential information
-credentials.json
+./credentials.json
 
 # Test related files
 .coverage

README.rst

@@ -38,20 +38,50 @@ Features
 .. _examples: https://snowexsql.readthedocs.io/en/latest/gallery/index.html
 
 
-Installing
-----------
-If you are just planning on using the database, then only install the
-python package instructions below.
+Setup
+-----
 
-I just want to use it
----------------------
+Installing
+==========
 Install using pip:
 
 .. code-block::
 
     pip install snowexsql
 
-I want data fast
+Configuring the database connection
+===================================
+Using this library requires setting up the database connection credentials.
+There are two options to do this:
+
+* Set database connection URL via ``SNOWEX_DB_CONNECTION`` environment variable
+  Example:
+
+.. code-block:: bash
+
+    export SNOWEX_DB_CONNECTION="user:password@127.0.0.1/db_name"
+
+* Point to a credentials JSON file via ``SNOWEX_DB_CREDENTIALS`` environment variable
+  Example
+
+.. code-block:: bash
+
+    export SNOWEX_DB_CREDENTIALS="/path/to/credentials.json"
+
+
+`Sample JSON file <./credentials.json.sample>`_:
+
+.. code-block:: json
+
+  {
+    "address": "localhost",
+    "db_name": "snowexdb",
+    "username": "user",
+    "password": "password"
+  }
+
+
+Accessing the SnowEx data
 -----------------
 A programmatic API has been created for fast and standard
 access to Point and Layer data. There are two examples_ covering the
@@ -69,24 +99,13 @@ detailed description.
     )
     print(df.head())
 
-I need help
+
+Getting help
 ------------
 Jump over to `our discussion forum <https://github.com/SnowEx/snowexsql/discussions>`_ 
 and get help from our community.
 
 
-I want to contribute
----------------------
-Thank you for the interest!
-
-Our community follows the |Contributor Covenant|
-
-.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg
-  :target: code_of_conduct.md
-.. _contribution guide: https://snowexsql.readthedocs.io/en/latest/community/contributing.html
-
-Have a look at our `contribution guide`_ and see the many ways to get involved!
-
 Documentation
 -------------
 
@@ -111,6 +130,42 @@ last image submitted to GitHub.
 
   make docs
 
+
+I want to contribute
+---------------------
+Thank you for the interest!
+
+Our community follows the |Contributor Covenant|
+
+.. |Contributor Covenant| image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg
+  :target: code_of_conduct.md
+.. _contribution guide: https://snowexsql.readthedocs.io/en/latest/community/contributing.html
+
+Have a look at our `contribution guide`_ and see the many ways to get involved!
+
+Testing
+=======
+To run the test suite locally requires having a running instance of PostgreSQL.
+The test suite is configured to run against these credentials:
+
+.. code-block::
+
+  builder:db_builder@localhost/test
+
+This requires a running database on ``localhost`` where the user ``builder`` has access
+to the ``test`` database with the password ``db_builder``.
+
+It is possible to set a custom host and database via the ``SNOWEX_TEST_DB`` environment
+variable. Example that would connect to a server on ``my.host`` and the database
+``snowex_test``:
+
+.. code-block:: bash
+
+    export SNOWEX_TEST_DB="my_host/snowex_test"
+
+More on connection strings to PostgreSQL can be found with the
+`official documentation <https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING>`_.
+
 DOI
 ---
 .. |HW22| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.7618102.svg 

credentials.json.sample

@@ -1,14 +1,6 @@
 {
-  "production": {
-    "address": "localhost",
-    "db_name": "snowexsql",
-    "username": "builder",
-    "password": "db_builder"
-  },
-  "tests": {
-    "address": "localhost",
-    "db_name": "test",
-    "username": "builder",
-    "password": "db_builder"
-  }
+  "address": "localhost",
+  "db_name": "test",
+  "username": "user",
+  "password": "password"
 }

snowexsql/db.py

@@ -1,21 +1,18 @@
 """
-This module contains tool used directly regarding the database. This includes
-getting a session, initializing the database, getting table attributes, etc.
+This module handles loading the database connection information and creating
+a session.
 """
 
 import json
 import os
 from contextlib import contextmanager
-from pathlib import Path
 
 from snowexsql.tables.base import Base
 from sqlalchemy import MetaData, create_engine
 from sqlalchemy.orm import sessionmaker
 
-# The default credentials file name
-CREDENTIAL_FILE="credentials.json"
 # This library requires a postgres dialect and the psycopg2 driver
-DB_CONNECTION_PROTOCOL = 'postgresql+psycopg2://'
+DB_CONNECTION_PROTOCOL = "postgresql+psycopg2://"
 # Always create a Session in UTC time
 DB_CONNECTION_OPTIONS = {"options": "-c timezone=UTC"}
 
@@ -43,19 +40,28 @@ def load_credentials(credentials_path=None):
     Returns:
         Dictionary - Credential information
     """
-    if credentials_path is None:
-        credentials_path = Path(__file__).parent.parent / CREDENTIAL_FILE
+    credentials = None
 
-    with open(credentials_path) as file:
-        credentials = json.load(file)
+    if credentials_path is not None:
+        with open(credentials_path) as file:
+            credentials = json.load(file)
+    elif os.getenv("SNOWEX_DB_CREDENTIALS"):
+        with open(os.getenv("SNOWEX_DB_CREDENTIALS")) as file:
+            credentials = json.load(file)
 
-        if os.getenv('SNOWEXSQL_TESTS', False):
-            return credentials['tests']
-        else:
-            return credentials['production']
+    if credentials is None:
+        raise FileNotFoundError(
+            "File to credentials was not provided. Please use the following options: "
+            "* Pass the file path to this method "
+            "* Set the SNOWEX_DB_CREDENTIALS environment variable to a JSON file"
+            "* Set the SNOWEX_DB_CONNECTION environment variable. "
+            "  Example: user:password@127.0.0.1/db_name"
+        )
 
+    return credentials
 
-def db_connection_string(credentials_path=None):
+
+def db_connection_string(credentials_path:str = None):
     """
     Construct a connection info string for SQLAlchemy database
 
@@ -65,16 +71,22 @@ def db_connection_string(credentials_path=None):
     Returns:
         String - DB connection
     """
-    credentials = load_credentials(credentials_path)
-
     db = DB_CONNECTION_PROTOCOL
-    db += f"{credentials['username']}:{credentials['password']}"\
-          f"@{credentials['address']}/{credentials['db_name']}"
+
+    if os.getenv("SNOWEX_DB_CONNECTION"):
+        credentials = os.getenv("SNOWEX_DB_CONNECTION")
+        db += credentials
+    else:
+        credentials = load_credentials(credentials_path)
+        db += (
+            f"{credentials['username']}:{credentials['password']}"
+            f"@{credentials['address']}/{credentials['db_name']}"
+        )
 
     return db
 
 
-def get_db(credentials_path=None, return_metadata=False):
+def get_db(credentials_path: str = None, return_metadata: bool = False):
     """
     Returns the DB engine, MetaData, and session object
 
@@ -126,8 +138,8 @@ def get_table_attributes(DataCls):
     Returns a list of all the table columns to be used for each entry
     """
 
-    valid_attributes = [att for att in dir(DataCls) if att[0] != '_']
+    valid_attributes = [att for att in dir(DataCls) if att[0] != "_"]
 
     # Drop ID as it is (should) never provided
-    valid_attributes = [v for v in valid_attributes if v != 'id']
+    valid_attributes = [v for v in valid_attributes if v != "id"]
     return valid_attributes

tests/conftest.py

@@ -1,22 +1,33 @@
 import os
 from contextlib import contextmanager
+from pathlib import Path
 
 import pytest
 import snowexsql
 from pytest_factoryboy import register
-from snowexsql.db import (
-    DB_CONNECTION_OPTIONS, db_connection_string, initialize
-)
+from snowexsql.db import DB_CONNECTION_OPTIONS, db_connection_string, initialize
 from sqlalchemy import create_engine
+from tests.factories import (
+   CampaignFactory,
+   DOIFactory,
+   InstrumentFactory,
+   LayerDataFactory,
+   LayerDensityFactory,
+   LayerTemperatureFactory,
+   MeasurementTypeFactory,
+   ObserverFactory,
+   PointDataFactory,
+   PointObservationFactory,
+   SiteFactory,
+)
+
 from tests import SESSION
-from tests.factories import (CampaignFactory, DOIFactory, InstrumentFactory,
-                             LayerDataFactory, LayerDensityFactory,
-                             LayerTemperatureFactory, MeasurementTypeFactory,
-                             ObserverFactory, PointDataFactory,
-                             PointObservationFactory, SiteFactory)
 
-# Environment variable to load the correct credentials
-os.environ['SNOWEXSQL_TESTS'] = 'True'
+# Environment variable to load the custom DB test credentials
+if os.getenv("SNOWEX_TEST_DB") is None:
+    os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@localhost/test"
+else:
+    os.environ["SNOWEX_DB_CONNECTION"] = "builder:db_builder@" + os.getenv("SNOWEX_TEST_DB")
 
 # Make factories available to tests
 register(CampaignFactory)
@@ -35,6 +46,12 @@
 def session_maker():
    return
 
+
+@pytest.fixture(scope="session")
+def data_dir():
+    return Path(__file__).parent.joinpath("data").absolute().resolve()
+
+
 # Add this factory to a test if you would like to debug the SQL statement
 # It will print the query from the BaseDataset.from_filter() method
 @pytest.fixture(scope='session')

tests/data/credentials.json

@@ -0,0 +1,6 @@
+{
+  "address": "localhost",
+  "db_name": "test",
+  "username": "builder",
+  "password": "db_builder"
+}