Opencdms cloud (#51)

faysal-ishtiaq · web-flow · commit b46e2cd275ad · 2022-07-11T01:50:08.000+06:00
* add security scheme to climsoft_app

* add oauth2 password rewuest form to swagger

* remove unnecessary files

* refactor deployment code

* cleanup and screenshots

* update readme

* minor refactor

* adapt changes with opencdms-cloud

* refactor docker compose for prod
diff --git a/.github/workflows/deploy_stable.yml b/.github/workflows/deploy_stable.yml
@@ -20,18 +20,22 @@ jobs:
             HOST_FQDN: ${{secrets.STABLE_HOST_FQDN}}
             AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
             AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
-            AUTH_ENABLED: "True"
+            AUTH_ENABLED: true
+            CLIMSOFT_API_ENABLED: true
+            MCH_API_ENABLED: true
+            SURFACE_API_ENABLED: true
+            PYGEOAPI_ENABLED: true
 
         run: |
           echo "$PRIVATE_KEY" > private_key && chmod 600 private_key
-          ssh -o StrictHostKeyChecking=no -o SendEnv=AUTH_ENABLED -o SendEnv=HOST_FQDN -o SendEnv=AWS_ACCESS_KEY_ID -o SendEnv=AWS_SECRET_ACCESS_KEY -i private_key ${USERNAME}@${HOSTNAME} '
+          ssh -o StrictHostKeyChecking=no -o SendEnv=AUTH_ENABLED -o SendEnv=HOST_FQDN -o SendEnv=AWS_ACCESS_KEY_ID -o SendEnv=AWS_SECRET_ACCESS_KEY -o SendEnv=CLIMSOFT_API_ENABLED -o SendEnv=SURFACE_API_ENABLED -o SendEnv=MCH_API_ENABLED -o SendEnv=PYGEOAPI_ENABLED -i private_key ${USERNAME}@${HOSTNAME} '
             cd /home/ubuntu/opencdms-test-data
             git pull origin main
             cd /home/ubuntu/opencdms-api
             git pull origin main
-            docker-compose down -v --remove-orphans
+            docker-compose -f docker-compose.prod.yml down -v --remove-orphans
             docker-compose -f ../opencdms-test-data/docker-compose.yml down -v --remove-orphans
             docker-compose -f ../opencdms-test-data/docker-compose.yml up -d postgresql mysql oracle
             sleep 15
-            docker-compose up -d --build
+            docker-compose -f docker-compose.prod.yml up -d --build
           '
diff --git a/Dockerfile b/Dockerfile
@@ -27,7 +27,6 @@ RUN pip install -r requirements.txt
 
 COPY ./scripts ./scripts
 COPY entrypoint.sh ./entrypoint.sh
-COPY init_climsoft_db.py ./init_climsoft_db.py
 COPY mch.dbn ./mch.dbn
 COPY MCHtablasycampos.def ./MCHtablasycampos.def
 
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -0,0 +1,150 @@
+version: "3"
+
+services:
+  opencdms_api:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: opencdms_api
+    ports:
+      - "5070:5000"
+    env_file:
+      - .env
+    environment:
+      - PYGEOAPI_CONFIG=/code/pygeoapi-config.yml
+      - PYGEOAPI_OPENAPI=/code/pygeoapi-openapi.yml
+      - PYTHONPATH=/code/surface/api
+      - CLIMSOFT_DATABASE_URI=mysql+mysqldb://root:password@mysql:3306/mariadb_climsoft_test_db_v4
+      - CLIMSOFT_SECRET_KEY=climsoft-secret-key
+      - AUTH_DB_URI=postgresql+psycopg2://dba:dba@opencdms_surface_db:5432/surface
+      - SURFACE_DB_NAME=surface
+      - SURFACE_DB_USER=dba
+      - SURFACE_DB_PASSWORD=dba
+      - SURFACE_DB_HOST=opencdms_surfacedb
+      - SURFACE_DB_PORT=5432
+      - MCH_DB_PORT=3306
+      - MCH_DB_HOST=mch-english
+      - MCH_DB_NAME=mch
+      - MCH_DB_PASSWORD=password
+      - MCH_DB_USER=root
+      - APP_SECRET=app-secret
+      - SURFACE_SECRET_KEY=surface-secret-key
+      - SECRET_KEY=secret-key
+      - TIMESCALEDB_TELEMETRY=off
+      - PGDATA=/var/lib/postgresql/data/pgdata
+      - SURFACE_DATA_DIR=/home/surface/surface_data/shared
+      - SURFACE_DB_ENGINE=django.contrib.gis.db.backends.postgis
+      - SURFACE_BROKER_URL=redis://redis:6379/0
+      - SURFACE_DJANGO_DEBUG=False
+      - LOGIN_REDIRECT_URL=/wx/stations/map/
+      - LOGOUT_REDIRECT_URL=/accounts/login/
+      - LRGS_EXECUTABLE_PATH=/surface/LrgsClient/bin/getDcpMessages
+      - LRGS_SERVER_HOST=lrgseddn1.cr.usgs.gov
+      - LRGS_SERVER_PORT=16003
+      - LRGS_USER=belnms
+      - LRGS_PASSWORD=BWSNlrgs2016!
+      - LRGS_CS_FILE_PATH=/data/search_parameters.cs
+      - LRGS_MAX_INTERVAL=719
+      - ENTL_PRIMARY_SERVER_HOST=107.23.152.248
+      - ENTL_PRIMARY_SERVER_PORT=2324
+      - ENTL_SECONDARY_SERVER_HOST=107.23.135.182
+      - ENTL_SECONDARY_SERVER_PORT=2324
+      - ENTL_PARTNER_ID=2B6FDADE-CA7F-443A-AD79-2FF21CEF4857
+      - EMAIL_HOST=smtp.gmail.com
+      - EMAIL_HOST_USER=test_email_host
+      - EMAIL_HOST_PASSWORD=test_email_password
+      - EMAIL_PORT=587
+      - TIMEZONE_NAME=America/Belize
+      - TIMEZONE_OFFSET=-360
+      - INMET_HOURLY_DATA_URL=
+      - INMET_DAILY_DATA_BASE_PATH=
+      - MAP_LATITUDE=17.302212
+      - MAP_LONGITUDE=-88.429595
+      - MAP_ZOOM=8
+      - SPATIAL_ANALYSIS_INITIAL_LATITUDE=15.8469375676
+      - SPATIAL_ANALYSIS_INITIAL_LONGITUDE=-89.227
+      - SPATIAL_ANALYSIS_FINAL_LATITUDE=18.5299822047
+      - SPATIAL_ANALYSIS_FINAL_LONGITUDE=-87.485
+      - SPATIAL_ANALYSIS_SHAPE_FILE_PATH=/surface/static/images/blz_shape.png
+      - STATION_MAP_WIND_SPEED_ID=51
+      - STATION_MAP_WIND_GUST_ID=53
+      - STATION_MAP_WIND_DIRECTION_ID=56
+      - STATION_MAP_TEMP_MAX_ID=16
+      - STATION_MAP_TEMP_MIN_ID=14
+      - STATION_MAP_TEMP_AVG_ID=10
+      - STATION_MAP_ATM_PRESSURE_ID=60
+      - STATION_MAP_PRECIPITATION_ID=0
+      - STATION_MAP_RELATIVE_HUMIDITY_ID=30
+      - STATION_MAP_SOLAR_RADIATION_ID=72
+      - STATION_MAP_FILTER_WATERSHED=1
+      - STATION_MAP_FILTER_REGION=1
+      - STATION_MAP_FILTER_COMMUNICATION=1
+      - SURFACE_API_ENABLED=$SURFACE_API_ENABLED
+      - CLIMSOFT_API_ENABLED=$CLIMSOFT_API_ENABLED
+      - MCH_API_ENABLED=$MCH_API_ENABLED
+      - PYGEOAPI_ENABLED=$PYGEOAPI_ENABLED
+      - DEFAULT_USERNAME=admin
+      - DEFAULT_PASSWORD=password123
+      - HOST_FQDN=$HOST_FQDN
+      - CLIMSOFT_AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+      - CLIMSOFT_AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+      - AUTH_ENABLED=true
+    depends_on:
+      - opencdms_surfacedb
+    volumes:
+      - ./src:/code/src
+      - ./climsoft_uploads:/climsoft_uploads
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.fastapi.rule=Host(`$HOST_FQDN`)"
+      - "traefik.http.routers.fastapi.tls=true"
+      - "traefik.http.routers.fastapi.tls.certresolver=letsencrypt"
+    networks:
+      - opencdms-test-data_opencdms
+    command:
+      [
+        "uvicorn",
+        "src.opencdms_api.main:app",
+        "--host",
+        "0.0.0.0",
+        "--port",
+        "5000",
+        "--reload",
+        "--proxy-headers",
+      ]
+
+  opencdms_surfacedb:
+    image: timescale/timescaledb-postgis:2.3.0-pg13
+    container_name: opencdms_surface_db
+    volumes:
+      - opencdms_surface_data:/var/lib/postgresql/data
+    ports:
+      - "65432:5432"
+    environment:
+      - POSTGRES_PASSWORD=dba
+      - POSTGRES_DB=surface
+      - POSTGRES_USER=dba
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "1M"
+        max-file: "10"
+    networks:
+      - opencdms-test-data_opencdms
+
+  traefik:
+    image: traefik:latest
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "$PWD/traefik/traefik.toml:/etc/traefik/traefik.toml"
+
+volumes:
+  opencdms_surface_data:
+
+networks:
+  opencdms-test-data_opencdms:
+    external: true
+    driver: bridge
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -82,6 +82,7 @@ services:
       - SURFACE_API_ENABLED=true
       - CLIMSOFT_API_ENABLED=true
       - MCH_API_ENABLED=true
+      - PYGEOAPI_ENABLED=true
       - DEFAULT_USERNAME=admin
       - DEFAULT_PASSWORD=password123
       - HOST_FQDN=$HOST_FQDN
diff --git a/src/opencdms_api/config.py b/src/opencdms_api/config.py
@@ -18,6 +18,7 @@ class Settings(BaseSettings):
     SURFACE_API_ENABLED: bool
     CLIMSOFT_API_ENABLED: bool
     MCH_API_ENABLED: bool
+    PYGEOAPI_ENABLED: bool
 
     DEFAULT_USERNAME: str
     DEFAULT_PASSWORD: str
diff --git a/src/opencdms_api/main.py b/src/opencdms_api/main.py
@@ -25,10 +25,6 @@
 from opencdms.models.climsoft import v4_1_1_core as climsoft_models
 from src.opencdms_api.middleware import get_authorized_climsoft_user
 from climsoft_api.api import api_routers
-from fastapi.security import OAuth2PasswordBearer
-
-
-oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth")
 
 
 # load controllers
@@ -46,26 +42,39 @@ def get_app():
     )
     climsoft_app = get_climsoft_app()
 
-    if settings.SURFACE_API_ENABLED is True:
+    if settings.SURFACE_API_ENABLED:
         surface_wsgi_app = WSGIMiddleware(surface_application)
         app.mount("/surface", surface_wsgi_app)
 
-    if settings.MCH_API_ENABLED is True:
+    if settings.MCH_API_ENABLED:
         mch_wsgi_app = WSGIMiddleware(mch_api_application)
-        app.mount("/mch", AuthMiddleWare(mch_wsgi_app))
-
-    if settings.CLIMSOFT_API_ENABLED is True:
-        for r in api_routers:
-            climsoft_app.include_router(
-                **r.dict(),
-                dependencies=[
-                    Depends(oauth2_scheme)
-                ]
-            )
+        if settings.AUTH_ENABLED:
+            app.mount("/mch", AuthMiddleWare(mch_wsgi_app))
+        else:
+            app.mount("/mch", mch_wsgi_app)
+
+    if settings.CLIMSOFT_API_ENABLED:
+        if settings.AUTH_ENABLED:
+            for r in api_routers:
+                climsoft_app.include_router(
+                    **r.dict(),
+                    dependencies=[
+                        Depends(get_authorized_climsoft_user)
+                    ]
+                )
+        else:
+            for r in api_routers:
+                climsoft_app.include_router(
+                    **r.dict()
+                )
         app.mount("/climsoft", climsoft_app)
 
-    pygeoapi_wsgi_app = WSGIMiddleware(pygeoapi_app)
-    app.mount("/pygeoapi", AuthMiddleWare(pygeoapi_wsgi_app))
+    if settings.PYGEOAPI_ENABLED:
+        pygeoapi_wsgi_app = WSGIMiddleware(pygeoapi_app)
+        if settings.AUTH_ENABLED:
+            app.mount("/pygeoapi", AuthMiddleWare(pygeoapi_wsgi_app))
+        else:
+            app.mount("/pygeoapi", pygeoapi_wsgi_app)
 
     app.include_router(router)
 
diff --git a/src/opencdms_api/middleware.py b/src/opencdms_api/middleware.py
@@ -15,6 +15,10 @@
 from src.opencdms_api.schema import CurrentUserSchema
 from opencdms.models.climsoft import v4_1_1_core as climsoft_models
 from fastapi import Header, Depends
+from fastapi.security import OAuth2PasswordBearer
+
+
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/auth")
 
 
 def get_user(username: str) -> Optional[CurrentUserSchema]:
@@ -121,11 +125,8 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send):
 
 def get_authorized_climsoft_user(
     request: Request,
-    authorization: str = Header(None)
+    token: str = Depends(oauth2_scheme)
 ):
-    scheme, token = get_authorization_scheme_param(authorization)
-    if scheme.lower() != "bearer":
-        raise HTTPException(401, "Invalid authorization header scheme")
     try:
         claims = jwt.decode(token, settings.SURFACE_SECRET_KEY)
     except JWTError:
