To1 pr0701 layer import button

layers/admin.py

@@ -1,14 +1,18 @@
 from collections import OrderedDict
+import json
 from dal import autocomplete
 from django.contrib import admin
+from django.apps import apps
 from django.contrib.contenttypes.admin import GenericTabularInline
 from django.contrib.contenttypes.models import ContentType
 from django.conf import settings
 from django import forms
 from django.forms.models import inlineformset_factory
 from django.db import transaction
+from django.core.exceptions import PermissionDenied
 from django.http import JsonResponse
-from django.urls import path
+from django.shortcuts import redirect, render
+from django.urls import path, reverse
 from django.utils import timezone
 from django.utils.decorators import method_decorator
 from django.utils.html import format_html
@@ -19,6 +23,8 @@
 import os
 from queryset_sequence import QuerySetSequence
 import requests
+from .fixture_contract import validate_node_shape
+from .fixture_import import import_fixture_rows
 from .models import *
 
 # MP-Layers is meant to fully replace MP-Data-Manager, but several pieces are still
@@ -34,6 +40,11 @@
     admin.site.unregister(DMLayer)
 except Exception as e:
     pass
+try:
+    from data_manager.models import AttributeInfo as DMAttributeInfo
+    admin.site.unregister(DMAttributeInfo)
+except Exception as e:
+    pass
 
 # Register your models here.
 
@@ -881,6 +892,56 @@ def export_layer_details(self, request, queryset):
 
 
 class LayerAdmin(ImportExportMixin, nested_admin.NestedModelAdmin):
+    fixture_import_session_key = "layers.fixture_import_rows"
+
+    def values_match(self, current_value, new_value):
+        from datetime import datetime
+        if current_value == new_value:
+            return True
+        if isinstance(current_value, datetime):
+            new_time = datetime.fromisoformat(new_value)
+            return current_value == new_time
+        if isinstance(current_value, uuid.UUID):
+            return current_value == uuid.UUID(new_value)
+        return False
+
+    def _fixture_preview_rows(self, rows):
+        preview_rows = []
+        for row in rows:
+            model_label = row['model']
+            uuid_value = row['uuid']
+            existing_record = None
+            if uuid_value:
+                model = apps.get_model(model_label)
+                manager = getattr(model, 'all_objects', model._base_manager)
+                existing_record = manager.filter(uuid=uuid_value).first()
+
+            changes = []
+            if existing_record is not None:
+                for field_name, new_value in row['fields'].items():
+                    current_value = getattr(existing_record, field_name)
+                    if not self.values_match(current_value, new_value):
+                        changes.append({
+                            'name': field_name,
+                            'current_value': current_value,
+                            'new_value': new_value,
+                        })
+
+            if existing_record is not None:
+                action = f'Update existing record: "{existing_record}"'
+            elif uuid_value:
+                action = 'Create new record'
+            else:
+                action = 'Create or merge relationship record'
+
+            preview_rows.append({
+                'model': model_label,
+                'uuid': uuid_value,
+                'action': action,
+                'changes': changes,
+            })
+        return preview_rows
+
     def get_parent_themes(self, obj):
         # Fetch the ContentType for the Layer model
         content_type = ContentType.objects.get_for_model(obj)
@@ -1028,6 +1089,7 @@ def formfield_for_manytomany(self, db_field, request=None, **kwargs):
 
     add_form_template = os.path.join(CURRENT_DIR, 'templates', 'admin', 'layers', 'Layer', 'change_form.html')
     change_form_template = os.path.join(CURRENT_DIR, 'templates', 'admin', 'layers', 'Layer', 'change_form.html')
+    change_list_template = os.path.join(CURRENT_DIR, 'templates', 'admin', 'layers', 'Layer', 'change_list.html')
 
     def change_view(self, request, object_id, form_url='', extra_context={}):
         extra_context['CATALOG_TECHNOLOGY'] = settings.CATALOG_TECHNOLOGY
@@ -1205,11 +1267,92 @@ def get_form(self, request, obj=None, **kwargs):
     def get_urls(self):
         urls = super().get_urls()
         custom_urls = [
+            path(
+                'import-fixture/',
+                self.admin_site.admin_view(self.import_fixture),
+                name='layers_layer_import_fixture',
+            ),
             path('get-layer-list/', self.admin_site.admin_view(self.get_layer_list), name='get-layer-list'),
             path('update-layer-status/<int:layer_id>/', self.admin_site.admin_view(self.update_layer_status), name='update-layer-status'),
         ]
         return custom_urls + urls
 
+    def import_fixture(self, request):
+        import json
+        if not self.has_change_permission(request):
+            raise PermissionDenied
+
+        changelist_url = reverse('admin:layers_layer_changelist')
+        context = {
+            **self.admin_site.each_context(request),
+            'opts': self.model._meta,
+            'title': 'Import layer fixture',
+            'changelist_url': changelist_url,
+        }
+
+        if request.method == 'POST' and 'cancel' in request.POST:
+            request.session.pop(self.fixture_import_session_key, None)
+            return redirect(changelist_url)
+
+        if request.method == 'POST' and 'confirm' in request.POST:
+            rows = request.session.get(self.fixture_import_session_key)
+            if rows is None:
+                context['error'] = 'No validated fixture is available to import.'
+                return render(request, 'admin/layers/Layer/import_layer_fixture.html', context)
+
+            try:
+                result = import_fixture_rows(
+                    rows,
+                    dry_run=False,
+                    associate_all_sites=True,
+                    missing_ref_policy='error',
+                    duplicate_uuid_policy='error',
+                )
+            except ValueError as error:
+                context['error'] = str(error)
+                context['fixture_rows'] = rows
+                context['preview_rows'] = self._fixture_preview_rows(rows)
+                return render(request, 'admin/layers/Layer/import_layer_fixture.html', context)
+
+            request.session.pop(self.fixture_import_session_key, None)
+            self.message_user(request, 'Imported {} fixture rows.'.format(result['imported']))
+            return redirect(changelist_url)
+
+        if request.method == 'POST':
+            fixture_file = request.FILES.get('fixture_file')
+            if fixture_file is None:
+                context['error'] = 'Choose a fixture JSON file to import.'
+                return render(request, 'admin/layers/Layer/import_layer_fixture.html', context)
+
+            try:
+                rows = json.loads(fixture_file.read().decode('utf-8'))
+                if not isinstance(rows, list):
+                    raise ValueError('Fixture JSON must contain a list of rows.')
+                for row in rows:
+                    validate_node_shape(row)
+            except (UnicodeDecodeError, json.JSONDecodeError, ValueError) as error:
+                context['error'] = 'Upload valid JSON fixture data: {}'.format(error)
+                return render(request, 'admin/layers/Layer/import_layer_fixture.html', context)
+
+            try:
+                result = import_fixture_rows(
+                    rows,
+                    dry_run=True,
+                    associate_all_sites=True,
+                    missing_ref_policy='error',
+                    duplicate_uuid_policy='error',
+                )
+            except ValueError as error:
+                context['error'] = str(error)
+                return render(request, 'admin/layers/Layer/import_layer_fixture.html', context)
+
+            request.session[self.fixture_import_session_key] = rows
+            context['fixture_rows'] = rows
+            context['preview_result'] = result
+            context['preview_rows'] = self._fixture_preview_rows(rows)
+
+        return render(request, 'admin/layers/Layer/import_layer_fixture.html', context)
+
     def http_status(self, obj):
         return format_html(
             '<span class="http-status" data-layer-id="{}" data-url="{}" data-name="{}" data-status="{}">{}</span>',
@@ -1271,7 +1414,11 @@ class LookupInfoAdmin(admin.ModelAdmin):
 #     list_display = ('parent_theme', 'content_type', 'object_id', 'order')
 #     form = ChildOrderForm
 
+class AttributeInfoAdmin(admin.ModelAdmin):
+    list_display = ('field_name', 'display_name', 'precision', 'order', 'preserve_format')
+
 admin.site.register(Theme, ThemeAdmin)
 admin.site.register(Layer, LayerAdmin)
 admin.site.register(LookupInfo, LookupInfoAdmin)
+admin.site.register(AttributeInfo, AttributeInfoAdmin)
 # admin.site.register(ChildOrder, ChildOrderAdmin)

layers/fixture_import.py

@@ -15,6 +15,7 @@
     NODE_FIELDS_KEY,
     NODE_MODEL_KEY,
     NODE_RELATIONS_KEY,
+    NODE_SOURCE_PK_KEY,
     NODE_UUID_KEY,
     normalize_uuid,
 )
@@ -163,7 +164,11 @@ def _execute_import():
             row_obj = model_manager.filter(uuid=row_uuid).first()
             is_new = row_obj is None
             if is_new:
-                row_obj = model_class(uuid=row_uuid)
+                source_pk = row.get(NODE_SOURCE_PK_KEY)
+                create_kwargs = {"uuid": row_uuid}
+                if source_pk is not None and not model_manager.filter(pk=source_pk).exists():
+                    create_kwargs["pk"] = source_pk
+                row_obj = model_class(**create_kwargs)
 
             _apply_fields(row_obj, row_fields)
             row_obj.save()

layers/templates/admin/layers/Layer/change_list.html

@@ -0,0 +1,8 @@
+{% extends "admin/change_list.html" %}
+
+{% block object-tools-items %}
+  <li>
+    <a href="{% url 'admin:layers_layer_import_fixture' %}">Import layer fixture</a>
+  </li>
+  {{ block.super }}
+{% endblock %}

layers/templates/admin/layers/Layer/import_layer_fixture.html

@@ -0,0 +1,60 @@
+{% extends "admin/layers/import_fixture.html" %}
+
+{% block fixture_import_content %}
+  <div id="content-main">
+    <h1>{{ title }}</h1>
+
+    {% if error %}
+      <p class="errornote">{{ error }}</p>
+    {% endif %}
+
+    {% if fixture_rows %}
+      <p>Validated {{ fixture_rows|length }} fixture row{{ fixture_rows|length|pluralize }}.</p>
+      {% for row in preview_rows %}
+        <section>
+          <h2>{{ row.model }}</h2>
+          <p>{{ row.action }}</p>
+          {% if row.changes %}
+            <table>
+              <tbody>
+                <tr>
+                  {% for change in row.changes %}
+                    <th scope="col">{{ change.name }}</th>
+                  {% endfor %}
+                </tr>
+                <tr>
+                  {% for change in row.changes %}
+                    <td>{{ change.current_value }}</td>
+                  {% endfor %}
+                </tr>
+                <tr>
+                  {% for change in row.changes %}
+                    <td>{{ change.new_value }}</td>
+                  {% endfor %}
+                </tr>
+              </tbody>
+            </table>
+          {% else %}
+            <p>No changes.</p>
+          {% endif %}
+        </section>
+      {% endfor %}
+      <form method="post">
+        {% csrf_token %}
+        <div class="submit-row">
+          <input type="submit" name="confirm" value="Confirm">
+          <input class="cancel-button" type="submit" name="cancel" value="Cancel">
+        </div>
+      </form>
+    {% else %}
+      <form method="post" enctype="multipart/form-data">
+        {% csrf_token %}
+        <p>
+          <label for="id_fixture_file">Layer fixture JSON</label>
+          <input id="id_fixture_file" name="fixture_file" type="file" accept="application/json,.json" required>
+        </p>
+        <input type="submit" value="Preview import">
+      </form>
+    {% endif %}
+  </div>
+{% endblock %}

layers/templates/admin/layers/import_fixture.html

@@ -0,0 +1,25 @@
+{% extends "admin/base_site.html" %}
+
+{% block extrastyle %}
+  {{ block.super }}
+  <style>
+    .submit-row input.cancel-button {
+      background: var(--delete-button-bg);
+      border-radius: 4px;
+      padding: 0.625rem 0.9375rem;
+      line-height: 0.9375rem;
+      color: var(--button-fg);
+    }
+
+    .submit-row input.cancel-button:focus,
+    .submit-row input.cancel-button:hover,
+    .submit-row input.cancel-button:active {
+      background: var(--delete-button-hover-bg);
+      text-decoration: none;
+    }
+  </style>
+{% endblock %}
+
+{% block content %}
+  {% block fixture_import_content %}{% endblock %}
+{% endblock %}

layers/tests/test_fixture_import.py

@@ -103,6 +103,26 @@ def test_source_pk_collision_with_different_uuid_creates_new_record(self):
         existing_layer.refresh_from_db()
         self.assertEqual(existing_layer.name, "Existing")
 
+    def test_unused_source_pk_is_preserved_when_creating_record(self):
+        self._require_importer()
+
+        new_uuid = uuid4()
+        source_pk = 999999
+        fixture_rows = [
+            build_node(
+                model="layers.layer",
+                source_pk=source_pk,
+                uuid_value=new_uuid,
+                fields=self._layer_fields("Preserved Source PK"),
+                relations={},
+            )
+        ]
+
+        import_fixture_rows(fixture_rows, **self._import_kwargs())
+
+        imported_layer = Layer.all_objects.get(uuid=new_uuid)
+        self.assertEqual(imported_layer.pk, source_pk)
+
     def test_second_pass_resolves_relations_by_uuid_not_source_pk(self):
         """As name suggests - ensure 2nd pass uses UUIDs for reference, not just PK or 'id'."""
         self._require_importer()