From ae6754298c133ddf666b5ec3caa3107d6e337638 Mon Sep 17 00:00:00 2001 From: Jas Kalayan Date: Wed, 12 Aug 2026 15:43:08 +0100 Subject: [PATCH 1/2] moved extracted data to tmpdir from session, updated tests to reflect this --- biosimdb_interface/form/extract.py | 4 +-- biosimdb_interface/form/upload.py | 57 +++++++++++++++++++++++++----- biosimdb_interface/form/webform.py | 21 ++++++++--- tests/test_form/test_upload.py | 38 ++++++++++++++++++-- tests/test_form/test_webform.py | 10 ++++-- 5 files changed, 109 insertions(+), 21 deletions(-) diff --git a/biosimdb_interface/form/extract.py b/biosimdb_interface/form/extract.py index 244bce5..f93f4eb 100644 --- a/biosimdb_interface/form/extract.py +++ b/biosimdb_interface/form/extract.py @@ -11,7 +11,7 @@ import shutil from biosim_extractor.metadata.populatemetadata import MetadataPopulator -from flask import jsonify, request, session +from flask import jsonify, request from . import form_bp from .utils import make_upload_tmpdir @@ -86,7 +86,7 @@ def extract_metadata(): result, validation_errors = extract_files_validate(topo_path, traj_files) # Keep authoritative extracted payload on the server - session["extracted_metadata"] = result + # session["extracted_metadata"] = result if len(validation_errors) > 0: return jsonify( diff --git a/biosimdb_interface/form/upload.py b/biosimdb_interface/form/upload.py index a185d0b..815065c 100644 --- a/biosimdb_interface/form/upload.py +++ b/biosimdb_interface/form/upload.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import glob import json import os import shutil @@ -12,6 +11,42 @@ from .invenio import run_record_upload from .utils import fill_invenio_metadata, form_to_json, make_upload_tmpdir +PENDING_FORM_FILENAME = "pending_form_data.json" +PENDING_UPLOADS_FILENAME = "pending_uploads.json" +SIM_METADATA_FILENAME = "simulation_metadata.json" + +INTERNAL_TMP_FILENAMES = { + PENDING_FORM_FILENAME, + PENDING_UPLOADS_FILENAME, + "metadata.json", +} + + +def _pending_form_path(tmpdir): + return os.path.join(tmpdir, PENDING_FORM_FILENAME) + + +def _pending_uploads_path(tmpdir): + return os.path.join(tmpdir, PENDING_UPLOADS_FILENAME) + + +def _flatten_saved_files(saved_files): + return [p for paths in saved_files.values() for p in paths] + + +def _load_pending_upload_paths(tmpdir): + path = _pending_uploads_path(tmpdir) + with open(path) as f: + saved_files = json.load(f) + files = [p for p in _flatten_saved_files(saved_files) if os.path.isfile(p)] + + # Optional: keep this if simulation_metadata.json must be included in record files + sim_meta_path = os.path.join(tmpdir, SIM_METADATA_FILENAME) + if os.path.isfile(sim_meta_path): + files.append(sim_meta_path) + + return files + def _save_request_files(tmpdir): """Save uploaded request files into a temporary directory grouped by role. @@ -104,20 +139,25 @@ def save_pending_submission(json_form=None): added before writing. Side effects: - session["pending_form_data"]: Set to submitted form data (dict of lists). session["pending_files_dir"]: Set to temporary directory path containing - uploaded files and optional ``simulation_metadata.json``. + uploaded files plus persisted JSON payloads used after login. """ tmpdir = make_upload_tmpdir("biosimdb_pending_") - _, file_meta = _save_files_and_extract_metadata(tmpdir) + saved_files, file_meta = _save_files_and_extract_metadata(tmpdir) + + # Persist exact user-uploaded paths for later allowlist upload + with open(_pending_uploads_path(tmpdir), "w") as f: + json.dump(saved_files, f) if json_form is not None: json_form["files"] = file_meta - json_path = os.path.join(tmpdir, "simulation_metadata.json") + json_path = os.path.join(tmpdir, SIM_METADATA_FILENAME) with open(json_path, "w") as f: json.dump(json_form, f, indent=2) - session["pending_form_data"] = request.form.to_dict(flat=False) + with open(_pending_form_path(tmpdir), "w") as f: + json.dump(request.form.to_dict(flat=False), f) + session["pending_files_dir"] = tmpdir @@ -137,9 +177,8 @@ def prepare_for_invenio(form_data, tmpdir): metadata_path = os.path.join(tmpdir, "metadata.json") with open(metadata_path, "w") as f: json.dump(invenio_data, f, indent=2) - file_paths = [ - p for p in glob.glob(os.path.join(tmpdir, "*")) if p != metadata_path - ] + + file_paths = _load_pending_upload_paths(tmpdir) _, draft_id = _data_collections_upload(metadata_path, file_paths) finally: shutil.rmtree(tmpdir, ignore_errors=True) diff --git a/biosimdb_interface/form/webform.py b/biosimdb_interface/form/webform.py index 07b2bfd..3f81d03 100644 --- a/biosimdb_interface/form/webform.py +++ b/biosimdb_interface/form/webform.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import json import os import requests @@ -67,7 +68,7 @@ def webform(): if action == "save": json_form["files"] = extract_uploaded_file_metadata() - # NOTE: note used yet, could be used to validate extracted fields are matching what is returned from json_form + # NOTE: note used yet, could be used to validate extracted fields are matching what is returned from json_form, cookie size increases though # extracted = session.get("extracted_metadata") biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "") @@ -117,7 +118,11 @@ def resume_submit(): """ if not session.get("access_token"): return redirect(url_for("login.login")) - if not session.get("pending_form_data") or not session.get("pending_files_dir"): + tmpdir = session.get("pending_files_dir") + pending_form_path = ( + os.path.join(tmpdir, "pending_form_data.json") if tmpdir else None + ) + if not tmpdir or not pending_form_path or not os.path.isfile(pending_form_path): flash("No pending submission found.", "warning") return redirect(url_for("form.webform")) return render_template("form/loading.html") @@ -131,13 +136,20 @@ def do_submit(): Clears pending session data after upload and renders the success page with the record URL. """ - form_data = session.pop("pending_form_data", None) tmpdir = session.pop("pending_files_dir", None) - if not form_data or not tmpdir: + if not tmpdir: flash("No pending submission found. Please submit again.", "warning") return redirect(url_for("form.webform")) + pending_form_path = os.path.join(tmpdir, "pending_form_data.json") + if not os.path.isfile(pending_form_path): + flash("No pending submission found. Please submit again.", "warning") + return redirect(url_for("form.webform")) + + with open(pending_form_path) as f: + form_data = json.load(f) + flat_form = ImmutableMultiDict( [(k, v) for k, vals in form_data.items() for v in vals] ) @@ -166,7 +178,6 @@ def do_submit(): return redirect(url_for("form.webform")) # success: now clear pending state - session.pop("pending_form_data", None) session.pop("pending_files_dir", None) BASE_URL = current_app.config["BASE_URL"] diff --git a/tests/test_form/test_upload.py b/tests/test_form/test_upload.py index 8c70a4a..e88e0ea 100644 --- a/tests/test_form/test_upload.py +++ b/tests/test_form/test_upload.py @@ -1,3 +1,4 @@ +import json import os import tempfile from unittest.mock import patch @@ -8,7 +9,11 @@ def test_prepare_for_invenio(app): """prepare_for_invenio writes metadata.json, calls _data_collections_upload, cleans up.""" tmpdir = tempfile.mkdtemp() - open(os.path.join(tmpdir, "traj.xtc"), "w").close() # dummy file + traj_path = os.path.join(tmpdir, "traj.xtc") + open(traj_path, "w").close() # dummy file + # New allowlist manifest required by _load_pending_upload_paths() + with open(os.path.join(tmpdir, "pending_uploads.json"), "w") as f: + json.dump({"trajectory": [traj_path]}, f) form_data = ImmutableMultiDict([("simulation[1][name]", "test")]) @@ -22,6 +27,30 @@ def test_prepare_for_invenio(app): draft_id = prepare_for_invenio(form_data, tmpdir) assert draft_id == "draft-abc" assert not os.path.exists(tmpdir) # cleaned up + _, files_path = mock_upload.call_args.args + assert traj_path in files_path + + +def test_load_pending_upload_paths_uses_manifest_and_includes_sim_metadata(app): + tmpdir = tempfile.mkdtemp() + traj_path = os.path.join(tmpdir, "traj.xtc") + top_path = os.path.join(tmpdir, "top.pdb") + sim_meta_path = os.path.join(tmpdir, "simulation_metadata.json") + + open(traj_path, "w").close() + open(top_path, "w").close() + open(sim_meta_path, "w").close() + + with open(os.path.join(tmpdir, "pending_uploads.json"), "w") as f: + json.dump({"trajectory": [traj_path], "topology": [top_path]}, f) + with app.app_context(): + from biosimdb_interface.form.upload import _load_pending_upload_paths + + files = _load_pending_upload_paths(tmpdir) + + assert traj_path in files + assert top_path in files + assert sim_meta_path in files def test_save_pending_submission(client): @@ -46,10 +75,13 @@ def test_save_pending_submission(client): def test_do_submit_calls_invenio(client): """Submission triggers Invenio upload with correct args.""" + tmpdir = tempfile.mkdtemp() + with open(os.path.join(tmpdir, "pending_form_data.json"), "w") as f: + json.dump({"simulation_name": ["test"]}, f) + with client.session_transaction() as sess: sess["access_token"] = "fake-token" - sess["pending_form_data"] = {"simulation_name": ["test"]} - sess["pending_files_dir"] = "/tmp/fake_pending" + sess["pending_files_dir"] = tmpdir with ( patch("biosimdb_interface.form.webform.invite_user") as mock_invite, diff --git a/tests/test_form/test_webform.py b/tests/test_form/test_webform.py index fb87654..e1d39f4 100644 --- a/tests/test_form/test_webform.py +++ b/tests/test_form/test_webform.py @@ -1,3 +1,6 @@ +import json +import os +import tempfile from unittest.mock import patch @@ -51,9 +54,12 @@ def test_submit_with_token_renders_loading(client): def test_resume_submit_with_pending_data(client): """resume_submit renders loading page when session has pending submission.""" + tmpdir = tempfile.mkdtemp() + with open(os.path.join(tmpdir, "pending_form_data.json"), "w") as f: + json.dump({"x": ["y"]}, f) + with client.session_transaction() as sess: sess["access_token"] = "tok" - sess["pending_form_data"] = {"x": ["y"]} - sess["pending_files_dir"] = "/tmp/fake" + sess["pending_files_dir"] = tmpdir response = client.get("/resume_submit") assert response.status_code == 200 From 40b07d72071fe54bde4f681d1d1cb5397a7438b3 Mon Sep 17 00:00:00 2001 From: Jas Kalayan Date: Wed, 12 Aug 2026 15:57:12 +0100 Subject: [PATCH 2/2] add docstrings --- biosimdb_interface/form/upload.py | 76 +++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 18 deletions(-) diff --git a/biosimdb_interface/form/upload.py b/biosimdb_interface/form/upload.py index 815065c..abb4843 100644 --- a/biosimdb_interface/form/upload.py +++ b/biosimdb_interface/form/upload.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +"""Upload helpers for deferred BioSimDB submission and Invenio transfer.""" import json import os @@ -23,18 +24,57 @@ def _pending_form_path(tmpdir): + """Return the path to the persisted pending form payload JSON. + + Args: + tmpdir (str): Temporary directory containing pending submission artifacts. + + Returns: + str: Path to file. + """ return os.path.join(tmpdir, PENDING_FORM_FILENAME) def _pending_uploads_path(tmpdir): + """Return the path to the persisted uploaded-files manifest JSON. + + Args: + tmpdir (str): Temporary directory containing pending submission artifacts. + + Returns: + str: Path to file. + """ return os.path.join(tmpdir, PENDING_UPLOADS_FILENAME) def _flatten_saved_files(saved_files): + """Flatten a role-to-path mapping into a single file path list. + + Args: + saved_files (dict[str, list[str]]): Mapping of file role to saved file paths. + + Returns: + list[str]: Flattened list of saved file paths. + """ return [p for paths in saved_files.values() for p in paths] def _load_pending_upload_paths(tmpdir): + """Load allowed upload file paths for deferred submission. + + Reads the persisted uploads manifest and returns existing files only. + If simulation_metadata.json exists, it is appended to the upload list. + + Args: + tmpdir (str): Temporary directory containing pending submission artifacts. + + Returns: + list[str]: File paths that should be uploaded to Invenio. + + Raises: + FileNotFoundError: If pending_uploads.json is missing. + json.JSONDecodeError: If pending_uploads.json is not valid JSON. + """ path = _pending_uploads_path(tmpdir) with open(path) as f: saved_files = json.load(f) @@ -123,24 +163,20 @@ def _data_collections_upload(metadata_path, files_path): def save_pending_submission(json_form=None): - """Save uploaded files and form data for deferred post-login submission. - - Writes uploaded request files to a new temporary directory, computes file - metadata from those saved files, and stores pending submission state in the - Flask session so submission can resume after OAuth login. + """Persist uploaded files and form payload for post-login submission resume. - If ``json_form`` is provided, this function attaches the computed file - metadata under ``json_form["files"]`` and writes the result to - ``simulation_metadata.json`` in the temporary directory. + Saves uploaded request files into a temp directory, writes a manifest of + allowed upload paths, optionally writes simulation_metadata.json, and stores + the form payload as pending_form_data.json. Args: - json_form: Optional converted/validated BioSim metadata dictionary to - persist alongside uploaded files. When provided, file metadata is - added before writing. + json_form (dict | None): Validated BioSim metadata to persist. When + provided, file metadata is attached at json_form["files"] before + writing simulation_metadata.json. - Side effects: - session["pending_files_dir"]: Set to temporary directory path containing - uploaded files plus persisted JSON payloads used after login. + Side Effects: + Writes JSON artifacts under tmpdir. + Sets session["pending_files_dir"]. """ tmpdir = make_upload_tmpdir("biosimdb_pending_") saved_files, file_meta = _save_files_and_extract_metadata(tmpdir) @@ -162,14 +198,18 @@ def save_pending_submission(json_form=None): def prepare_for_invenio(form_data, tmpdir): - """Convert form data and upload files from tmpdir to Invenio. Cleans up tmpdir. + """Create Invenio metadata and upload allowlisted files from tmpdir. Args: - form_data: Flat form data (ImmutableMultiDict or similar) from the webform submission. - tmpdir: Path to temporary directory containing uploaded simulation files. + form_data (ImmutableMultiDict | Mapping): Submitted webform payload. + tmpdir (str): Temporary directory containing pending files and manifests. Returns: - draft_id: The Invenio draft record ID of the created upload. + draft_id (str): Created Invenio draft record ID. + + Side Effects: + Writes metadata.json in tmpdir. + Deletes tmpdir on exit. """ try: json_form = form_to_json(form_data)