28 bug session cookie too large

biosimdb_interface/form/extract.py

@@ -11,7 +11,7 @@
 import shutil
 
 from biosim_extractor.metadata.populatemetadata import MetadataPopulator
-from flask import jsonify, request, session
+from flask import jsonify, request
 
 from . import form_bp
 from .utils import make_upload_tmpdir
@@ -86,7 +86,7 @@ def extract_metadata():
             result, validation_errors = extract_files_validate(topo_path, traj_files)
 
             # Keep authoritative extracted payload on the server
-            session["extracted_metadata"] = result
+            # session["extracted_metadata"] = result
 
             if len(validation_errors) > 0:
                 return jsonify(

biosimdb_interface/form/upload.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
+"""Upload helpers for deferred BioSimDB submission and Invenio transfer."""
 
-import glob
 import json
 import os
 import shutil
@@ -12,6 +12,81 @@
 from .invenio import run_record_upload
 from .utils import fill_invenio_metadata, form_to_json, make_upload_tmpdir
 
+PENDING_FORM_FILENAME = "pending_form_data.json"
+PENDING_UPLOADS_FILENAME = "pending_uploads.json"
+SIM_METADATA_FILENAME = "simulation_metadata.json"
+
+INTERNAL_TMP_FILENAMES = {
+    PENDING_FORM_FILENAME,
+    PENDING_UPLOADS_FILENAME,
+    "metadata.json",
+}
+
+
+def _pending_form_path(tmpdir):
+    """Return the path to the persisted pending form payload JSON.
+
+    Args:
+    tmpdir (str): Temporary directory containing pending submission artifacts.
+
+    Returns:
+    str: Path to file.
+    """
+    return os.path.join(tmpdir, PENDING_FORM_FILENAME)
+
+
+def _pending_uploads_path(tmpdir):
+    """Return the path to the persisted uploaded-files manifest JSON.
+
+    Args:
+    tmpdir (str): Temporary directory containing pending submission artifacts.
+
+    Returns:
+    str: Path to file.
+    """
+    return os.path.join(tmpdir, PENDING_UPLOADS_FILENAME)
+
+
+def _flatten_saved_files(saved_files):
+    """Flatten a role-to-path mapping into a single file path list.
+
+    Args:
+    saved_files (dict[str, list[str]]): Mapping of file role to saved file paths.
+
+    Returns:
+    list[str]: Flattened list of saved file paths.
+    """
+    return [p for paths in saved_files.values() for p in paths]
+
+
+def _load_pending_upload_paths(tmpdir):
+    """Load allowed upload file paths for deferred submission.
+
+    Reads the persisted uploads manifest and returns existing files only.
+    If simulation_metadata.json exists, it is appended to the upload list.
+
+    Args:
+    tmpdir (str): Temporary directory containing pending submission artifacts.
+
+    Returns:
+    list[str]: File paths that should be uploaded to Invenio.
+
+    Raises:
+    FileNotFoundError: If pending_uploads.json is missing.
+    json.JSONDecodeError: If pending_uploads.json is not valid JSON.
+    """
+    path = _pending_uploads_path(tmpdir)
+    with open(path) as f:
+        saved_files = json.load(f)
+    files = [p for p in _flatten_saved_files(saved_files) if os.path.isfile(p)]
+
+    # Optional: keep this if simulation_metadata.json must be included in record files
+    sim_meta_path = os.path.join(tmpdir, SIM_METADATA_FILENAME)
+    if os.path.isfile(sim_meta_path):
+        files.append(sim_meta_path)
+
+    return files
+
 
 def _save_request_files(tmpdir):
     """Save uploaded request files into a temporary directory grouped by role.
@@ -88,58 +163,62 @@ def _data_collections_upload(metadata_path, files_path):
 
 
 def save_pending_submission(json_form=None):
-    """Save uploaded files and form data for deferred post-login submission.
-
-    Writes uploaded request files to a new temporary directory, computes file
-    metadata from those saved files, and stores pending submission state in the
-    Flask session so submission can resume after OAuth login.
+    """Persist uploaded files and form payload for post-login submission resume.
 
-    If ``json_form`` is provided, this function attaches the computed file
-    metadata under ``json_form["files"]`` and writes the result to
-    ``simulation_metadata.json`` in the temporary directory.
+    Saves uploaded request files into a temp directory, writes a manifest of
+    allowed upload paths, optionally writes simulation_metadata.json, and stores
+    the form payload as pending_form_data.json.
 
     Args:
-        json_form: Optional converted/validated BioSim metadata dictionary to
-        persist alongside uploaded files. When provided, file metadata is
-        added before writing.
-
-    Side effects:
-        session["pending_form_data"]: Set to submitted form data (dict of lists).
-        session["pending_files_dir"]: Set to temporary directory path containing
-        uploaded files and optional ``simulation_metadata.json``.
+    json_form (dict | None): Validated BioSim metadata to persist. When
+    provided, file metadata is attached at json_form["files"] before
+    writing simulation_metadata.json.
+
+    Side Effects:
+    Writes JSON artifacts under tmpdir.
+    Sets session["pending_files_dir"].
     """
     tmpdir = make_upload_tmpdir("biosimdb_pending_")
-    _, file_meta = _save_files_and_extract_metadata(tmpdir)
+    saved_files, file_meta = _save_files_and_extract_metadata(tmpdir)
+
+    # Persist exact user-uploaded paths for later allowlist upload
+    with open(_pending_uploads_path(tmpdir), "w") as f:
+        json.dump(saved_files, f)
 
     if json_form is not None:
         json_form["files"] = file_meta
-        json_path = os.path.join(tmpdir, "simulation_metadata.json")
+        json_path = os.path.join(tmpdir, SIM_METADATA_FILENAME)
         with open(json_path, "w") as f:
             json.dump(json_form, f, indent=2)
 
-    session["pending_form_data"] = request.form.to_dict(flat=False)
+    with open(_pending_form_path(tmpdir), "w") as f:
+        json.dump(request.form.to_dict(flat=False), f)
+
     session["pending_files_dir"] = tmpdir
 
 
 def prepare_for_invenio(form_data, tmpdir):
-    """Convert form data and upload files from tmpdir to Invenio. Cleans up tmpdir.
+    """Create Invenio metadata and upload allowlisted files from tmpdir.
 
     Args:
-        form_data: Flat form data (ImmutableMultiDict or similar) from the webform submission.
-        tmpdir: Path to temporary directory containing uploaded simulation files.
+    form_data (ImmutableMultiDict | Mapping): Submitted webform payload.
+    tmpdir (str): Temporary directory containing pending files and manifests.
 
     Returns:
-        draft_id: The Invenio draft record ID of the created upload.
+    draft_id (str): Created Invenio draft record ID.
+
+    Side Effects:
+    Writes metadata.json in tmpdir.
+    Deletes tmpdir on exit.
     """
     try:
         json_form = form_to_json(form_data)
         invenio_data = fill_invenio_metadata(json_form)
         metadata_path = os.path.join(tmpdir, "metadata.json")
         with open(metadata_path, "w") as f:
             json.dump(invenio_data, f, indent=2)
-        file_paths = [
-            p for p in glob.glob(os.path.join(tmpdir, "*")) if p != metadata_path
-        ]
+
+        file_paths = _load_pending_upload_paths(tmpdir)
         _, draft_id = _data_collections_upload(metadata_path, file_paths)
     finally:
         shutil.rmtree(tmpdir, ignore_errors=True)

biosimdb_interface/form/webform.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+import json
 import os
 
 import requests
@@ -67,7 +68,7 @@ def webform():
             if action == "save":
                 json_form["files"] = extract_uploaded_file_metadata()
 
-            # NOTE: note used yet, could be used to validate extracted fields are matching what is returned from json_form
+            # NOTE: note used yet, could be used to validate extracted fields are matching what is returned from json_form, cookie size increases though
             # extracted = session.get("extracted_metadata")
 
             biosimschema_path = os.getenv("BIOSIM_SCHEMA_PATH", "")
@@ -117,7 +118,11 @@ def resume_submit():
     """
     if not session.get("access_token"):
         return redirect(url_for("login.login"))
-    if not session.get("pending_form_data") or not session.get("pending_files_dir"):
+    tmpdir = session.get("pending_files_dir")
+    pending_form_path = (
+        os.path.join(tmpdir, "pending_form_data.json") if tmpdir else None
+    )
+    if not tmpdir or not pending_form_path or not os.path.isfile(pending_form_path):
         flash("No pending submission found.", "warning")
         return redirect(url_for("form.webform"))
     return render_template("form/loading.html")
@@ -131,13 +136,20 @@ def do_submit():
     Clears pending session data after upload and renders the success page
     with the record URL.
     """
-    form_data = session.pop("pending_form_data", None)
     tmpdir = session.pop("pending_files_dir", None)
 
-    if not form_data or not tmpdir:
+    if not tmpdir:
         flash("No pending submission found. Please submit again.", "warning")
         return redirect(url_for("form.webform"))
 
+    pending_form_path = os.path.join(tmpdir, "pending_form_data.json")
+    if not os.path.isfile(pending_form_path):
+        flash("No pending submission found. Please submit again.", "warning")
+        return redirect(url_for("form.webform"))
+
+    with open(pending_form_path) as f:
+        form_data = json.load(f)
+
     flat_form = ImmutableMultiDict(
         [(k, v) for k, vals in form_data.items() for v in vals]
     )
@@ -166,7 +178,6 @@ def do_submit():
         return redirect(url_for("form.webform"))
 
     # success: now clear pending state
-    session.pop("pending_form_data", None)
     session.pop("pending_files_dir", None)
 
     BASE_URL = current_app.config["BASE_URL"]

tests/test_form/test_upload.py

@@ -1,3 +1,4 @@
+import json
 import os
 import tempfile
 from unittest.mock import patch
@@ -8,7 +9,11 @@
 def test_prepare_for_invenio(app):
     """prepare_for_invenio writes metadata.json, calls _data_collections_upload, cleans up."""
     tmpdir = tempfile.mkdtemp()
-    open(os.path.join(tmpdir, "traj.xtc"), "w").close()  # dummy file
+    traj_path = os.path.join(tmpdir, "traj.xtc")
+    open(traj_path, "w").close()  # dummy file
+    # New allowlist manifest required by _load_pending_upload_paths()
+    with open(os.path.join(tmpdir, "pending_uploads.json"), "w") as f:
+        json.dump({"trajectory": [traj_path]}, f)
 
     form_data = ImmutableMultiDict([("simulation[1][name]", "test")])
 
@@ -22,6 +27,30 @@ def test_prepare_for_invenio(app):
             draft_id = prepare_for_invenio(form_data, tmpdir)
             assert draft_id == "draft-abc"
             assert not os.path.exists(tmpdir)  # cleaned up
+            _, files_path = mock_upload.call_args.args
+            assert traj_path in files_path
+
+
+def test_load_pending_upload_paths_uses_manifest_and_includes_sim_metadata(app):
+    tmpdir = tempfile.mkdtemp()
+    traj_path = os.path.join(tmpdir, "traj.xtc")
+    top_path = os.path.join(tmpdir, "top.pdb")
+    sim_meta_path = os.path.join(tmpdir, "simulation_metadata.json")
+
+    open(traj_path, "w").close()
+    open(top_path, "w").close()
+    open(sim_meta_path, "w").close()
+
+    with open(os.path.join(tmpdir, "pending_uploads.json"), "w") as f:
+        json.dump({"trajectory": [traj_path], "topology": [top_path]}, f)
+    with app.app_context():
+        from biosimdb_interface.form.upload import _load_pending_upload_paths
+
+        files = _load_pending_upload_paths(tmpdir)
+
+    assert traj_path in files
+    assert top_path in files
+    assert sim_meta_path in files
 
 
 def test_save_pending_submission(client):
@@ -46,10 +75,13 @@ def test_save_pending_submission(client):
 
 def test_do_submit_calls_invenio(client):
     """Submission triggers Invenio upload with correct args."""
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "pending_form_data.json"), "w") as f:
+        json.dump({"simulation_name": ["test"]}, f)
+
     with client.session_transaction() as sess:
         sess["access_token"] = "fake-token"
-        sess["pending_form_data"] = {"simulation_name": ["test"]}
-        sess["pending_files_dir"] = "/tmp/fake_pending"
+        sess["pending_files_dir"] = tmpdir
 
     with (
         patch("biosimdb_interface.form.webform.invite_user") as mock_invite,

tests/test_form/test_webform.py

@@ -1,3 +1,6 @@
+import json
+import os
+import tempfile
 from unittest.mock import patch
 
 
@@ -51,9 +54,12 @@ def test_submit_with_token_renders_loading(client):
 
 def test_resume_submit_with_pending_data(client):
     """resume_submit renders loading page when session has pending submission."""
+    tmpdir = tempfile.mkdtemp()
+    with open(os.path.join(tmpdir, "pending_form_data.json"), "w") as f:
+        json.dump({"x": ["y"]}, f)
+
     with client.session_transaction() as sess:
         sess["access_token"] = "tok"
-        sess["pending_form_data"] = {"x": ["y"]}
-        sess["pending_files_dir"] = "/tmp/fake"
+        sess["pending_files_dir"] = tmpdir
     response = client.get("/resume_submit")
     assert response.status_code == 200