diff --git a/layers/admin.py b/layers/admin.py index 77a33d9..9ddf9b4 100644 --- a/layers/admin.py +++ b/layers/admin.py @@ -1,14 +1,18 @@ from collections import OrderedDict +import json from dal import autocomplete from django.contrib import admin +from django.apps import apps from django.contrib.contenttypes.admin import GenericTabularInline from django.contrib.contenttypes.models import ContentType from django.conf import settings from django import forms from django.forms.models import inlineformset_factory from django.db import transaction +from django.core.exceptions import PermissionDenied from django.http import JsonResponse -from django.urls import path +from django.shortcuts import redirect, render +from django.urls import path, reverse from django.utils import timezone from django.utils.decorators import method_decorator from django.utils.html import format_html @@ -19,6 +23,8 @@ import os from queryset_sequence import QuerySetSequence import requests +from .fixture_contract import validate_node_shape +from .fixture_import import import_fixture_rows from .models import * # MP-Layers is meant to fully replace MP-Data-Manager, but several pieces are still @@ -34,6 +40,11 @@ admin.site.unregister(DMLayer) except Exception as e: pass +try: + from data_manager.models import AttributeInfo as DMAttributeInfo + admin.site.unregister(DMAttributeInfo) +except Exception as e: + pass # Register your models here. @@ -881,6 +892,56 @@ def export_layer_details(self, request, queryset): class LayerAdmin(ImportExportMixin, nested_admin.NestedModelAdmin): + fixture_import_session_key = "layers.fixture_import_rows" + + def values_match(self, current_value, new_value): + from datetime import datetime + if current_value == new_value: + return True + if isinstance(current_value, datetime): + new_time = datetime.fromisoformat(new_value) + return current_value == new_time + if isinstance(current_value, uuid.UUID): + return current_value == uuid.UUID(new_value) + return False + + def _fixture_preview_rows(self, rows): + preview_rows = [] + for row in rows: + model_label = row['model'] + uuid_value = row['uuid'] + existing_record = None + if uuid_value: + model = apps.get_model(model_label) + manager = getattr(model, 'all_objects', model._base_manager) + existing_record = manager.filter(uuid=uuid_value).first() + + changes = [] + if existing_record is not None: + for field_name, new_value in row['fields'].items(): + current_value = getattr(existing_record, field_name) + if not self.values_match(current_value, new_value): + changes.append({ + 'name': field_name, + 'current_value': current_value, + 'new_value': new_value, + }) + + if existing_record is not None: + action = f'Update existing record: "{existing_record}"' + elif uuid_value: + action = 'Create new record' + else: + action = 'Create or merge relationship record' + + preview_rows.append({ + 'model': model_label, + 'uuid': uuid_value, + 'action': action, + 'changes': changes, + }) + return preview_rows + def get_parent_themes(self, obj): # Fetch the ContentType for the Layer model content_type = ContentType.objects.get_for_model(obj) @@ -1028,6 +1089,7 @@ def formfield_for_manytomany(self, db_field, request=None, **kwargs): add_form_template = os.path.join(CURRENT_DIR, 'templates', 'admin', 'layers', 'Layer', 'change_form.html') change_form_template = os.path.join(CURRENT_DIR, 'templates', 'admin', 'layers', 'Layer', 'change_form.html') + change_list_template = os.path.join(CURRENT_DIR, 'templates', 'admin', 'layers', 'Layer', 'change_list.html') def change_view(self, request, object_id, form_url='', extra_context={}): extra_context['CATALOG_TECHNOLOGY'] = settings.CATALOG_TECHNOLOGY @@ -1205,11 +1267,92 @@ def get_form(self, request, obj=None, **kwargs): def get_urls(self): urls = super().get_urls() custom_urls = [ + path( + 'import-fixture/', + self.admin_site.admin_view(self.import_fixture), + name='layers_layer_import_fixture', + ), path('get-layer-list/', self.admin_site.admin_view(self.get_layer_list), name='get-layer-list'), path('update-layer-status//', self.admin_site.admin_view(self.update_layer_status), name='update-layer-status'), ] return custom_urls + urls + def import_fixture(self, request): + import json + if not self.has_change_permission(request): + raise PermissionDenied + + changelist_url = reverse('admin:layers_layer_changelist') + context = { + **self.admin_site.each_context(request), + 'opts': self.model._meta, + 'title': 'Import layer fixture', + 'changelist_url': changelist_url, + } + + if request.method == 'POST' and 'cancel' in request.POST: + request.session.pop(self.fixture_import_session_key, None) + return redirect(changelist_url) + + if request.method == 'POST' and 'confirm' in request.POST: + rows = request.session.get(self.fixture_import_session_key) + if rows is None: + context['error'] = 'No validated fixture is available to import.' + return render(request, 'admin/layers/Layer/import_layer_fixture.html', context) + + try: + result = import_fixture_rows( + rows, + dry_run=False, + associate_all_sites=True, + missing_ref_policy='error', + duplicate_uuid_policy='error', + ) + except ValueError as error: + context['error'] = str(error) + context['fixture_rows'] = rows + context['preview_rows'] = self._fixture_preview_rows(rows) + return render(request, 'admin/layers/Layer/import_layer_fixture.html', context) + + request.session.pop(self.fixture_import_session_key, None) + self.message_user(request, 'Imported {} fixture rows.'.format(result['imported'])) + return redirect(changelist_url) + + if request.method == 'POST': + fixture_file = request.FILES.get('fixture_file') + if fixture_file is None: + context['error'] = 'Choose a fixture JSON file to import.' + return render(request, 'admin/layers/Layer/import_layer_fixture.html', context) + + try: + rows = json.loads(fixture_file.read().decode('utf-8')) + if not isinstance(rows, list): + raise ValueError('Fixture JSON must contain a list of rows.') + for row in rows: + validate_node_shape(row) + except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error: + context['error'] = 'Upload valid JSON fixture data: {}'.format(error) + return render(request, 'admin/layers/Layer/import_layer_fixture.html', context) + + try: + result = import_fixture_rows( + rows, + dry_run=True, + associate_all_sites=True, + missing_ref_policy='error', + duplicate_uuid_policy='error', + ) + except ValueError as error: + context['error'] = str(error) + return render(request, 'admin/layers/Layer/import_layer_fixture.html', context) + + request.session[self.fixture_import_session_key] = rows + context['fixture_rows'] = rows + context['preview_result'] = result + context['preview_rows'] = self._fixture_preview_rows(rows) + + return render(request, 'admin/layers/Layer/import_layer_fixture.html', context) + def http_status(self, obj): return format_html( '{}', @@ -1271,7 +1414,11 @@ class LookupInfoAdmin(admin.ModelAdmin): # list_display = ('parent_theme', 'content_type', 'object_id', 'order') # form = ChildOrderForm +class AttributeInfoAdmin(admin.ModelAdmin): + list_display = ('field_name', 'display_name', 'precision', 'order', 'preserve_format') + admin.site.register(Theme, ThemeAdmin) admin.site.register(Layer, LayerAdmin) admin.site.register(LookupInfo, LookupInfoAdmin) +admin.site.register(AttributeInfo, AttributeInfoAdmin) # admin.site.register(ChildOrder, ChildOrderAdmin) diff --git a/layers/fixture_import.py b/layers/fixture_import.py index 042bef9..53305bb 100644 --- a/layers/fixture_import.py +++ b/layers/fixture_import.py @@ -15,6 +15,7 @@ NODE_FIELDS_KEY, NODE_MODEL_KEY, NODE_RELATIONS_KEY, + NODE_SOURCE_PK_KEY, NODE_UUID_KEY, normalize_uuid, ) @@ -163,7 +164,11 @@ def _execute_import(): row_obj = model_manager.filter(uuid=row_uuid).first() is_new = row_obj is None if is_new: - row_obj = model_class(uuid=row_uuid) + source_pk = row.get(NODE_SOURCE_PK_KEY) + create_kwargs = {"uuid": row_uuid} + if source_pk is not None and not model_manager.filter(pk=source_pk).exists(): + create_kwargs["pk"] = source_pk + row_obj = model_class(**create_kwargs) _apply_fields(row_obj, row_fields) row_obj.save() diff --git a/layers/templates/admin/layers/Layer/change_list.html b/layers/templates/admin/layers/Layer/change_list.html new file mode 100644 index 0000000..fd304ec --- /dev/null +++ b/layers/templates/admin/layers/Layer/change_list.html @@ -0,0 +1,8 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} +
  • + Import layer fixture +
    • + {{ block.super }} +{% endblock %} diff --git a/layers/templates/admin/layers/Layer/import_layer_fixture.html b/layers/templates/admin/layers/Layer/import_layer_fixture.html new file mode 100644 index 0000000..27b4c3c --- /dev/null +++ b/layers/templates/admin/layers/Layer/import_layer_fixture.html @@ -0,0 +1,60 @@ +{% extends "admin/layers/import_fixture.html" %} + +{% block fixture_import_content %} +
    +

    {{ title }}

    + + {% if error %} +

    {{ error }}

    + {% endif %} + + {% if fixture_rows %} +

    Validated {{ fixture_rows|length }} fixture row{{ fixture_rows|length|pluralize }}.

    + {% for row in preview_rows %} +
    +

    {{ row.model }}

    +

    {{ row.action }}

    + {% if row.changes %} + + + + {% for change in row.changes %} + + {% endfor %} + + + {% for change in row.changes %} + + {% endfor %} + + + {% for change in row.changes %} + + {% endfor %} + + +
    {{ change.name }}
    {{ change.current_value }}
    {{ change.new_value }}
    + {% else %} +

    No changes.

    + {% endif %} +
    + {% endfor %} +
    + {% csrf_token %} +
    + + +
    +
    + {% else %} +
    + {% csrf_token %} +

    + + +

    + +
    + {% endif %} +
    +{% endblock %} \ No newline at end of file diff --git a/layers/templates/admin/layers/import_fixture.html b/layers/templates/admin/layers/import_fixture.html new file mode 100644 index 0000000..dab3c1c --- /dev/null +++ b/layers/templates/admin/layers/import_fixture.html @@ -0,0 +1,25 @@ +{% extends "admin/base_site.html" %} + +{% block extrastyle %} + {{ block.super }} + +{% endblock %} + +{% block content %} + {% block fixture_import_content %}{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/layers/tests/test_fixture_import.py b/layers/tests/test_fixture_import.py index cda5c32..f2c03a5 100644 --- a/layers/tests/test_fixture_import.py +++ b/layers/tests/test_fixture_import.py @@ -103,6 +103,26 @@ def test_source_pk_collision_with_different_uuid_creates_new_record(self): existing_layer.refresh_from_db() self.assertEqual(existing_layer.name, "Existing") + def test_unused_source_pk_is_preserved_when_creating_record(self): + self._require_importer() + + new_uuid = uuid4() + source_pk = 999999 + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=source_pk, + uuid_value=new_uuid, + fields=self._layer_fields("Preserved Source PK"), + relations={}, + ) + ] + + import_fixture_rows(fixture_rows, **self._import_kwargs()) + + imported_layer = Layer.all_objects.get(uuid=new_uuid) + self.assertEqual(imported_layer.pk, source_pk) + def test_second_pass_resolves_relations_by_uuid_not_source_pk(self): """As name suggests - ensure 2nd pass uses UUIDs for reference, not just PK or 'id'.""" self._require_importer() diff --git a/layers/tests/test_layer_fixture_import_admin.py b/layers/tests/test_layer_fixture_import_admin.py new file mode 100644 index 0000000..c504a8a --- /dev/null +++ b/layers/tests/test_layer_fixture_import_admin.py @@ -0,0 +1,194 @@ +import json +from unittest.mock import patch +from uuid import uuid4 + +from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile +from django.test import TestCase +from django.urls import reverse + +from layers.fixture_contract import build_node +from layers.models import Layer + + +class LayerFixtureImportAdminTest(TestCase): + """PR07.01 contract tests for the Layer fixture import admin workflow.""" + + upload_url_name = "admin:layers_layer_import_fixture" + + def setUp(self): + self.upload_url = reverse(self.upload_url_name) + self.superuser = get_user_model().objects.create_superuser( + username="fixture-admin", + email="fixture-admin@example.com", + password="password", + ) + self.valid_rows = [ + build_node( + model="layers.layer", + source_pk=1001, + uuid_value=uuid4(), + fields={ + "name": "Uploaded Fixture Layer", + "layer_type": "WMS", + "slug_name": None, + "url": None, + }, + relations={}, + ) + ] + + def _fixture_file(self, rows=None, name="layers.json"): + return SimpleUploadedFile( + name, + json.dumps(rows if rows is not None else self.valid_rows).encode("utf-8"), + content_type="application/json", + ) + + def test_upload_view_requires_layer_change_permission(self): + response = self.client.get(self.upload_url) + self.assertEqual(response.status_code, 302) + + user = get_user_model().objects.create_user( + username="no-layer-permission", + password="password", + is_staff=True, + ) + self.client.force_login(user) + response = self.client.get(self.upload_url) + self.assertEqual(response.status_code, 403) + + @patch("layers.admin.import_fixture_rows") + def test_valid_upload_runs_dry_run_and_shows_confirmation(self, import_fixture_rows): + self.client.force_login(self.superuser) + import_fixture_rows.return_value = {"imported": 1, "dry_run": True} + + response = self.client.post( + self.upload_url, + {"fixture_file": self._fixture_file()}, + ) + + self.assertEqual(response.status_code, 200) + import_fixture_rows.assert_called_once_with( + self.valid_rows, + dry_run=True, + associate_all_sites=True, + missing_ref_policy="error", + duplicate_uuid_policy="error", + ) + self.assertContains(response, "Uploaded Fixture Layer") + self.assertContains(response, "Confirm") + self.assertEqual(Layer.all_objects.count(), 0) + + @patch("layers.admin.import_fixture_rows") + def test_preview_reports_uuid_matched_updates_and_changed_field_values( + self, + import_fixture_rows, + ): + self.client.force_login(self.superuser) + existing_layer = Layer.all_objects.create( + name="Current Layer Name", + layer_type="WMS", + url="https://current.example.test/wms", + ) + fixture_rows = [ + build_node( + model="layers.layer", + source_pk=1001, + uuid_value=existing_layer.uuid, + fields={ + "name": "Imported Layer Name", + "layer_type": "WMS", + "slug_name": None, + "url": "https://imported.example.test/wms", + }, + relations={}, + ) + ] + import_fixture_rows.return_value = {"imported": 1, "dry_run": True} + + response = self.client.post( + self.upload_url, + {"fixture_file": self._fixture_file(fixture_rows)}, + ) + + self.assertEqual(response.status_code, 200) + self.assertContains(response, "Update existing record") + self.assertContains(response, "name") + self.assertContains(response, "Current Layer Name") + self.assertContains(response, "Imported Layer Name") + self.assertContains(response, "url") + self.assertContains(response, "https://current.example.test/wms") + self.assertContains(response, "https://imported.example.test/wms") + + @patch("layers.admin.import_fixture_rows") + def test_invalid_json_displays_error_without_import(self, import_fixture_rows): + self.client.force_login(self.superuser) + invalid_file = SimpleUploadedFile( + "broken.json", + b"{not valid json", + content_type="application/json", + ) + + response = self.client.post(self.upload_url, {"fixture_file": invalid_file}) + + self.assertEqual(response.status_code, 200) + import_fixture_rows.assert_not_called() + self.assertContains(response, "valid JSON") + self.assertEqual(Layer.all_objects.count(), 0) + + @patch("layers.admin.import_fixture_rows") + def test_confirmation_executes_staged_fixture_not_posted_payload(self, import_fixture_rows): + self.client.force_login(self.superuser) + import_fixture_rows.return_value = {"imported": 1, "dry_run": True} + + preview_response = self.client.post( + self.upload_url, + {"fixture_file": self._fixture_file()}, + ) + self.assertEqual(preview_response.status_code, 200) + + import_fixture_rows.reset_mock() + import_fixture_rows.return_value = {"imported": 1, "dry_run": False} + tampered_rows = [ + build_node( + model="layers.layer", + source_pk=9999, + uuid_value=uuid4(), + fields={ + "name": "Tampered Layer", + "layer_type": "WMS", + "slug_name": None, + "url": None, + }, + relations={}, + ) + ] + + response = self.client.post( + self.upload_url, + {"confirm": "1", "fixture_file": self._fixture_file(tampered_rows)}, + ) + + self.assertEqual(response.status_code, 302) + import_fixture_rows.assert_called_once_with( + self.valid_rows, + dry_run=False, + associate_all_sites=True, + missing_ref_policy="error", + duplicate_uuid_policy="error", + ) + + @patch("layers.admin.import_fixture_rows") + def test_cancel_discards_staged_fixture_without_import(self, import_fixture_rows): + self.client.force_login(self.superuser) + import_fixture_rows.return_value = {"imported": 1, "dry_run": True} + + self.client.post(self.upload_url, {"fixture_file": self._fixture_file()}) + import_fixture_rows.reset_mock() + + response = self.client.post(self.upload_url, {"cancel": "1"}) + + self.assertRedirects(response, reverse("admin:layers_layer_changelist")) + import_fixture_rows.assert_not_called() + self.assertEqual(Layer.all_objects.count(), 0) \ No newline at end of file