Add except pd.errors.ParserError for uploading malformed CSV files and add some related tests

src/mavedb/routers/score_calibrations.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Optional
 
+import pandas as pd
 from fastapi import APIRouter, Depends, File, HTTPException, Query, UploadFile
 from sqlalchemy.orm import Session, selectinload
 
@@ -360,6 +361,10 @@ async def create_score_calibration_route(
             raise HTTPException(
                 status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values."
             )
+        except pd.errors.ParserError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Error parsing classes file: {e}. Ensure the file is a valid CSV."
+            )
 
         try:
             standardized_classes_df, index_column = validate_and_standardize_calibration_classes_dataframe(
@@ -386,6 +391,10 @@ async def create_score_calibration_route(
             raise HTTPException(
                 status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values."
             )
+        except pd.errors.ParserError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Error parsing controls file: {e}. Ensure the file is a valid CSV."
+            )
 
         try:
             calibration.controls = validate_and_standardize_calibration_controls_dataframe(db, score_set, controls_df)
@@ -577,6 +586,10 @@ async def modify_score_calibration_route(
             raise HTTPException(
                 status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values."
             )
+        except pd.errors.ParserError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Error parsing classes file: {e}. Ensure the file is a valid CSV."
+            )
 
         try:
             standardized_classes_df, index_column = validate_and_standardize_calibration_classes_dataframe(
@@ -604,6 +617,10 @@ async def modify_score_calibration_route(
             raise HTTPException(
                 status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values."
             )
+        except pd.errors.ParserError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Error parsing controls file: {e}. Ensure the file is a valid CSV."
+            )
 
         try:
             calibration_update.controls = validate_and_standardize_calibration_controls_dataframe(

src/mavedb/routers/score_sets.py

@@ -527,6 +527,10 @@ async def parse_score_set_variants_uploads(
             raise HTTPException(
                 status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values."
             )
+        except pd.errors.ParserError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Error parsing scores file: {e}. Ensure the file is a valid CSV."
+            )
     else:
         scores_df = None
 
@@ -538,6 +542,10 @@ async def parse_score_set_variants_uploads(
             raise HTTPException(
                 status_code=400, detail=f"Error decoding file: {e}. Ensure the file has correct values."
             )
+        except pd.errors.ParserError as e:
+            raise HTTPException(
+                status_code=400, detail=f"Error parsing counts file: {e}. Ensure the file is a valid CSV."
+            )
     else:
         counts_df = None
 

tests/routers/test_score_calibrations.py

@@ -11,6 +11,7 @@
 import json
 from unittest.mock import patch
 
+import pandas as pd
 from arq import ArqRedis
 from sqlalchemy import select
 
@@ -1443,6 +1444,51 @@ def test_cannot_create_score_calibration_when_csv_file_fails_decoding(
     assert "Error decoding file:" in str(error["detail"])
 
 
+@pytest.mark.parametrize(
+    "mock_publication_fetch",
+    [
+        [
+            {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER},
+            {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER},
+        ]
+    ],
+    indirect=["mock_publication_fetch"],
+)
+def test_cannot_create_score_calibration_when_csv_file_fails_parsing(
+    client, setup_router_db, session, data_provider, data_files, mock_publication_fetch
+):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set_with_mapped_variants(
+        client,
+        session,
+        data_provider,
+        experiment["urn"],
+        data_files / "scores.csv",
+    )
+
+    calibration_csv_path = data_files / "calibration_classes_by_urn.csv"
+    with (
+        open(calibration_csv_path, "rb") as class_file,
+        patch(
+            "mavedb.routers.score_calibrations.csv_data_to_df",
+            side_effect=pd.errors.ParserError("Error tokenizing data. C error: Expected 1 fields in line 17, saw 4"),
+        ),
+    ):
+        response = client.post(
+            "/api/v1/score-calibrations",
+            files={"classes_file": (calibration_csv_path.name, class_file, "text/csv")},
+            data={
+                "calibration_json": json.dumps(
+                    {"scoreSetUrn": score_set["urn"], **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_CLASS_BASED)}
+                ),
+            },
+        )
+
+    assert response.status_code == 400
+    error = response.json()
+    assert "Error parsing classes file:" in str(error["detail"])
+
+
 @pytest.mark.parametrize(
     "mock_publication_fetch",
     [
@@ -2013,6 +2059,57 @@ def test_cannot_update_score_calibration_when_csv_file_fails_decoding(
     assert "Error decoding file:" in str(error["detail"])
 
 
+@pytest.mark.parametrize(
+    "mock_publication_fetch",
+    [
+        [
+            {"dbName": "PubMed", "identifier": TEST_PUBMED_IDENTIFIER},
+            {"dbName": "bioRxiv", "identifier": TEST_BIORXIV_IDENTIFIER},
+        ]
+    ],
+    indirect=["mock_publication_fetch"],
+)
+def test_cannot_update_score_calibration_when_csv_file_fails_parsing(
+    client, setup_router_db, session, data_provider, data_files, mock_publication_fetch
+):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set_with_mapped_variants(
+        client,
+        session,
+        data_provider,
+        experiment["urn"],
+        data_files / "scores.csv",
+    )
+    calibration = create_test_score_calibration_in_score_set_via_client(
+        client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED)
+    )
+
+    calibration_csv_path = data_files / "calibration_classes_by_urn.csv"
+    with (
+        open(calibration_csv_path, "rb") as class_file,
+        patch(
+            "mavedb.routers.score_calibrations.csv_data_to_df",
+            side_effect=pd.errors.ParserError("Error tokenizing data. C error: Expected 1 fields in line 17, saw 4"),
+        ),
+    ):
+        response = client.put(
+            f"/api/v1/score-calibrations/{calibration['urn']}",
+            files={"classes_file": (calibration_csv_path.name, class_file, "text/csv")},
+            data={
+                "calibration_json": json.dumps(
+                    {
+                        "scoreSetUrn": score_set["urn"],
+                        **deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_CLASS_BASED),
+                    }
+                ),
+            },
+        )
+
+    assert response.status_code == 400
+    error = response.json()
+    assert "Error parsing classes file:" in str(error["detail"])
+
+
 @pytest.mark.parametrize(
     "mock_publication_fetch",
     [

tests/routers/test_score_set.py

@@ -11,6 +11,7 @@
 from unittest.mock import patch
 
 import jsonschema
+import pandas as pd
 import pytest
 from humps import camelize
 from sqlalchemy import select
@@ -3963,6 +3964,56 @@ def test_upload_a_non_utf8_file(session, client, setup_router_db, data_files):
     )
 
 
+def test_upload_a_malformed_scores_csv_file(session, client, setup_router_db, data_files):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    scores_csv_path = data_files / "scores.csv"
+    with (
+        open(scores_csv_path, "rb") as scores_file,
+        patch(
+            "mavedb.routers.score_sets.csv_data_to_df",
+            side_effect=pd.errors.ParserError("Error tokenizing data. C error: Expected 1 fields in line 17, saw 4"),
+        ),
+    ):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']}/variants/data",
+            files={"scores_file": (scores_csv_path.name, scores_file, "text/csv")},
+        )
+    assert response.status_code == 422
+    response_data = response.json()
+    assert "Error parsing scores file:" in response_data["detail"]
+    assert "Ensure the file is a valid CSV." in response_data["detail"]
+
+
+def test_upload_a_malformed_counts_csv_file(session, client, setup_router_db, data_files):
+    experiment = create_experiment(client)
+    score_set = create_seq_score_set(client, experiment["urn"])
+    scores_csv_path = data_files / "scores.csv"
+    counts_csv_path = data_files / "scores.csv"
+    with (
+        open(scores_csv_path, "rb") as scores_file,
+        open(counts_csv_path, "rb") as counts_file,
+        patch(
+            "mavedb.routers.score_sets.csv_data_to_df",
+            side_effect=[
+                pd.read_csv(scores_csv_path),
+                pd.errors.ParserError("Error tokenizing data. C error: Expected 1 fields in line 17, saw 4"),
+            ],
+        ),
+    ):
+        response = client.post(
+            f"/api/v1/score-sets/{score_set['urn']}/variants/data",
+            files={
+                "scores_file": (scores_csv_path.name, scores_file, "text/csv"),
+                "counts_file": (counts_csv_path.name, counts_file, "text/csv"),
+            },
+        )
+    assert response.status_code == 422
+    response_data = response.json()
+    assert "Error parsing counts file:" in response_data["detail"]
+    assert "Ensure the file is a valid CSV." in response_data["detail"]
+
+
 ########################################################################################################################
 # Score set download files
 ########################################################################################################################