Follow-ups of statistics work

.github/workflows/ci.yml

@@ -101,6 +101,8 @@ jobs:
           wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.6-8b5b400-linux64.zip"
           mkdir -p venv/lib/python3.11/site-packages
           unzip -d venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
+          rm /tmp/ifcopenshell_python.zip
+          make install-ifcopenshell-mvd IFCOPENSHELL_SITE_PACKAGES=venv/lib/python3.11/site-packages
 
       - name: Check Django config
         run: |

.github/workflows/ci_cd.yml

@@ -101,6 +101,8 @@ jobs:
           wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.6-8b5b400-linux64.zip"
           mkdir -p venv/lib/python3.11/site-packages
           unzip -d venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
+          rm /tmp/ifcopenshell_python.zip
+          make install-ifcopenshell-mvd IFCOPENSHELL_SITE_PACKAGES=venv/lib/python3.11/site-packages
 
       - name: Check Django config
         run: |

.github/workflows/playwright.yml

@@ -110,6 +110,7 @@ jobs:
           mkdir -p .dev/venv/lib/python3.11/site-packages
           unzip -o -d .dev/venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
           rm /tmp/ifcopenshell_python.zip
+          make install-ifcopenshell-mvd IFCOPENSHELL_SITE_PACKAGES=.dev/venv/lib/python3.11/site-packages
           # Verify installation
           ls -la .dev/venv/lib/python3.11/site-packages/
           echo "Checking for ifcopenshell installation:"

backend/Makefile

@@ -3,6 +3,8 @@
 VIRTUAL_ENV = .dev/venv
 PYTHON = $(VIRTUAL_ENV)/bin/python
 PIP = $(VIRTUAL_ENV)/bin/pip 
+IFCOPENSHELL_SITE_PACKAGES = $(VIRTUAL_ENV)/lib/python3.11/site-packages
+PYTHON_MVDXML_REF = support-4.x-graphviz-format-2
 
 none:
 	@echo "MAKE: Enter at least one target (venv, install, install-dev, start-backend, start-worker, clean)"
@@ -15,25 +17,38 @@ install: venv
 	$(PIP) install --upgrade pip
 	find . -name 'requirements.txt' -exec $(PIP) install -r {} \;
 	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.6-8b5b400-linux64.zip"
-	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
-	unzip -o -d $(VIRTUAL_ENV)/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
+	mkdir -p $(IFCOPENSHELL_SITE_PACKAGES)
+	unzip -o -d $(IFCOPENSHELL_SITE_PACKAGES) /tmp/ifcopenshell_python.zip
 	rm /tmp/ifcopenshell_python.zip
+	$(MAKE) install-ifcopenshell-mvd
 
 install-macos: venv
 	find . -name 'requirements.txt' -exec $(PIP) install -r {} \;
 	$(PIP) install -r requirements.txt
 	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.6-8b5b400-macos64.zip"
-	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
-	unzip -o -d $(VIRTUAL_ENV)/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
+	mkdir -p $(IFCOPENSHELL_SITE_PACKAGES)
+	unzip -o -d $(IFCOPENSHELL_SITE_PACKAGES) /tmp/ifcopenshell_python.zip
 	rm /tmp/ifcopenshell_python.zip
+	$(MAKE) install-ifcopenshell-mvd
 
 install-macos-m1: venv
 	find . -name 'requirements.txt' -exec $(PIP) install -r {} \;
 	$(PIP) install -r requirements.txt
 	wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.6-8b5b400-macosm164.zip"
-	mkdir -p $(VIRTUAL_ENV)/lib/python3.11/site-packages
-	unzip -o -d $(VIRTUAL_ENV)/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
+	mkdir -p $(IFCOPENSHELL_SITE_PACKAGES)
+	unzip -o -d $(IFCOPENSHELL_SITE_PACKAGES) /tmp/ifcopenshell_python.zip
 	rm /tmp/ifcopenshell_python.zip
+	$(MAKE) install-ifcopenshell-mvd
+
+.PHONY: install-ifcopenshell-mvd
+install-ifcopenshell-mvd:
+	test -d "$(IFCOPENSHELL_SITE_PACKAGES)/ifcopenshell"
+	mvd_tmp=$$(mktemp -d)
+	trap 'rm -rf "$$mvd_tmp"' EXIT
+	wget -O "$$mvd_tmp/python-mvdxml.tar.gz" "https://github.com/opensourceBIM/python-mvdxml/archive/refs/heads/$(PYTHON_MVDXML_REF).tar.gz"
+	rm -rf "$(IFCOPENSHELL_SITE_PACKAGES)/ifcopenshell/mvd"
+	mkdir -p "$(IFCOPENSHELL_SITE_PACKAGES)/ifcopenshell/mvd"
+	tar -xzf "$$mvd_tmp/python-mvdxml.tar.gz" --strip-components=1 -C "$(IFCOPENSHELL_SITE_PACKAGES)/ifcopenshell/mvd"
 
 fetch-modules:
 	cd ./apps && git submodule update --init --recursive

backend/apps/ifc_validation/admin.py

@@ -1,16 +1,19 @@
 import logging
+from urllib.parse import urlencode
 
 from django.urls import path
 from django.contrib import admin
 from django.contrib import messages
 from django.contrib.auth import get_permission_codename
 from django.contrib.auth.admin import UserAdmin
 from django.contrib.auth.models import User
+from django.core.exceptions import FieldError, PermissionDenied
 from django.http import HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import reverse
 from django.utils.translation import ngettext
 from django.utils.html import format_html
+from django.db import DatabaseError
 from django.db.models import F, Case, When, DurationField, Count
 from django.db.models.functions import Now
 from django import forms
@@ -20,6 +23,9 @@
 from apps.ifc_validation_models.models import ValidationTask
 from apps.ifc_validation_models.models import ValidationOutcome
 from apps.ifc_validation_models.models import Model
+from apps.ifc_validation_models.models import EntityCountHistogram
+from apps.ifc_validation_models.models import PsetCountHistogram
+from apps.ifc_validation_models.models import TemplateStatistic
 from apps.ifc_validation_models.models import ModelInstance
 from apps.ifc_validation_models.models import Company
 from apps.ifc_validation_models.models import AuthoringTool
@@ -36,6 +42,16 @@
 
 from core import utils
 from core.filters import AdvancedDateFilter
+from .statistics_query import (
+    StatisticsQueryClauseFormSet,
+    StatisticsQueryBuilder,
+    StatisticsSourceForm,
+    build_statistics_specification,
+    bind_statistics_query_form_data,
+    format_sql,
+    model_histogram_query,
+    statistics_query_ui_context,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -503,7 +519,7 @@ class ModelAdmin(BaseAdmin, NonAdminAddable):
         ('Auditing Information', {"classes": ("wide"), "fields": [("created",), ("updated")]})
     ]
 
-    list_display = ["id", "public_id", "file_name", "size_text", "authoring_tool_link", "schema", "mvd", "timestamp", "header_file_name", "is_signed", "created", "updated"]
+    list_display = ["id", "public_id", "file_name", "size_text", "authoring_tool_link", "schema", "mvd", "timestamp", "header_file_name", "is_signed", "histogram_link", "pset_histogram_link", "created", "updated"]
     readonly_fields = ["id", "public_id", "file", "file_name", "size", "size_text", "date", "schema", "mvd", "produced_by", "created", "updated", "status_schema_calculated"]
     date_hierarchy = "created"
 
@@ -515,7 +531,86 @@ class ModelAdmin(BaseAdmin, NonAdminAddable):
         ('date', AdvancedDateFilter), 
         ('created', AdvancedDateFilter)
     ]
-
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom = [
+            path(
+                "statistics/",
+                self.admin_site.admin_view(self.statistics_view),
+                name="ifc_validation_models_model_statistics",
+            ),
+        ]
+        return custom + urls
+
+    @admin.display(description="Entities")
+    def histogram_link(self, obj):
+        link = reverse("admin:ifc_validation_models_model_statistics")
+        link = f"{link}?{urlencode({'source': 'entity', 'model': obj.pk})}"
+        return format_html('<a href="{}">View</a>', link)
+
+    @admin.display(description="Property Sets")
+    def pset_histogram_link(self, obj):
+        link = reverse("admin:ifc_validation_models_model_statistics")
+        link = f"{link}?{urlencode({'source': 'pset', 'model': obj.pk})}"
+        return format_html('<a href="{}">View</a>', link)
+
+    def statistics_view(self, request):
+        if not self.has_view_permission(request):
+            raise PermissionDenied
+
+        preset_query = None
+        if (
+            request.method == "GET"
+            and request.GET.get("source") in {"entity", "pset"}
+            and request.GET.get("model")
+        ):
+            preset_query = model_histogram_query(
+                request.GET["source"],
+                request.GET["model"],
+            )
+        data = (
+            request.POST
+            if request.method == "POST"
+            else bind_statistics_query_form_data(preset_query) if preset_query else None
+        )
+        source_form = StatisticsSourceForm(data)
+        clause_formset = StatisticsQueryClauseFormSet(
+            data,
+            prefix="clauses",
+        )
+        result = None
+        query_error = ""
+        forms_are_valid = (
+            data is not None
+            and source_form.is_valid()
+            and clause_formset.is_valid()
+        )
+        if forms_are_valid:
+            try:
+                specification = build_statistics_specification(
+                    source_form.cleaned_data["source"],
+                    clause_formset,
+                )
+                result = StatisticsQueryBuilder(specification).execute()
+            except (DatabaseError, FieldError, RuntimeError, ValueError) as error:
+                query_error = str(error)
+
+        context = {
+            **self.admin_site.each_context(request),
+            "opts": self.model._meta,
+            "title": "Model statistics query builder",
+            "source_form": source_form,
+            "clause_formset": clause_formset,
+            "query_error": query_error,
+            "columns": result.columns if result else [],
+            "rows": result.rows if result else [],
+            "display_rows": result.display_rows if result else [],
+            "sql": result.sql if result else "",
+            **statistics_query_ui_context(),
+        }
+        return TemplateResponse(request, "admin/model_statistics.html", context)
+
     @admin.display(description="File Size", ordering='size')
     def size_text(self, obj):
 
@@ -569,6 +664,22 @@ class ModelInstanceAdmin(BaseAdmin, NonAdminAddable):
     show_full_result_count = False # do not use COUNT(*) twice
 
 
+class EntityCountHistogramAdmin(admin.ModelAdmin):
+    readonly_fields = ["entity_name"]
+
+    @admin.display(description="Entity name")
+    def entity_name(self, obj):
+        return obj.entity_name
+
+
+class PsetCountHistogramAdmin(admin.ModelAdmin):
+    readonly_fields = ["entity_name"]
+
+    @admin.display(description="Entity name")
+    def entity_name(self, obj):
+        return obj.entity_name
+
+
 class CompanyAdmin(BaseAdmin):
 
     fieldsets = [
@@ -900,20 +1011,7 @@ def get_row(field):
 
                 try:
                     qs = entry.build().apply(ValidationOutcome.objects.filter(pk=outcome_id))
-                    sql = str(qs.query)
-                    try:
-                        # This is most likely a transitive dependency from django, but
-                        # if somehow unavailable it doesn't matter
-                        import sqlparse
-                        sql = sqlparse.format(
-                            sql,
-                            reindent=True,
-                            keyword_case="upper",
-                            identifier_case=None,
-                        )
-                    except:
-                        pass
-                    result["sql"] = sql
+                    result["sql"] = format_sql(str(qs.query))
                 except Exception as e:
                     result["error"] = str(e)
         else:
@@ -946,6 +1044,9 @@ class WhiteListTestForm(forms.Form):
 admin.site.register(ValidationTask, ValidationTaskAdmin)
 admin.site.register(ValidationOutcome, ValidationOutcomeAdmin)
 admin.site.register(Model, ModelAdmin)
+admin.site.register(EntityCountHistogram, EntityCountHistogramAdmin)
+admin.site.register(PsetCountHistogram, PsetCountHistogramAdmin)
+admin.site.register(TemplateStatistic)
 admin.site.register(ModelInstance, ModelInstanceAdmin)
 admin.site.register(Company, CompanyAdmin)
 admin.site.register(AuthoringTool, AuthoringToolAdmin)

backend/apps/ifc_validation/checks/statistics/apply_mvd.py

@@ -0,0 +1,73 @@
+import sys
+from pathlib import Path
+
+import ifcopenshell
+from ifcopenshell.mvd import template
+
+
+TEMPLATES_DIR = Path(__file__).parent / "templates"
+
+
+def available_template_names(templates_dir=TEMPLATES_DIR):
+    return tuple(
+        markdown.name for markdown in sorted(Path(templates_dir).glob("*.md"))
+    )
+
+
+def json_value(value):
+    if isinstance(value, ifcopenshell.entity_instance):
+        return value.is_a()
+    if isinstance(value, (list, tuple)):
+        return [json_value(item) for item in value]
+    return value
+
+
+def extract_template_statistics(
+    file_or_path,
+    templates_dir=TEMPLATES_DIR,
+    template_names=None,
+):
+    model = (
+        file_or_path
+        if isinstance(file_or_path, ifcopenshell.file)
+        else ifcopenshell.open(file_or_path)
+    )
+    results = []
+
+    selected_template_names = (
+        None if template_names is None else set(template_names)
+    )
+    for markdown in sorted(Path(templates_dir).glob("*.md")):
+        if (
+            selected_template_names is not None
+            and markdown.name not in selected_template_names
+        ):
+            continue
+        concept = template.from_graphviz(markdown.read_text(encoding="utf-8"))
+        try:
+            focus_instances = model.by_type(concept.entity)
+        except RuntimeError:
+            focus_instances = ()
+
+        for focus in focus_instances:
+            rows = concept.extract(focus)
+            if not rows:
+                continue
+            graph = {
+                concept.binding_for(key) or key.attribute: json_value(value)
+                for row in rows
+                for key, value in row.items()
+            }
+            results.append({
+                "template": markdown.name,
+                "focus_step_id": focus.id(),
+                "focus_ifc_type": focus.is_a(),
+                "graph": graph,
+            })
+
+    return results
+
+
+if __name__ == "__main__":
+    for result in extract_template_statistics(sys.argv[1]):
+        print(result)

backend/apps/ifc_validation/checks/statistics/templates/BasisCurves_of_IfcPointByDistanceExpression.md

@@ -0,0 +1,14 @@
+# BasisCurves of IfcPointByDistanceExpression
+
+> SCOPE IFC4.3+
+
+Per the schema IfcPointByDistanceExpression.BasisCurve is of type IfcCurve
+which includes a broad range of subtypes, most of which do not make sense in
+the context of linear referencing.
+
+```
+concept {
+    IfcPointByDistanceExpression:BasisCurve -> IfcCurve
+    IfcPointByDistanceExpression:BasisCurve[binding="BasisCurve"]
+}
+```

backend/apps/ifc_validation/checks/statistics/templates/Usage_of_IfcPropertySetDefinitionSet.md

@@ -0,0 +1,13 @@
+# Usage of IfcPropertySetDefinitionSet
+
+> SCOPE IFC4+
+
+IFC4 introduced a mechanism by which the objectified relationship for property
+set association obtained a select type between a single set and a set of sets.
+This template selects the second category.
+
+```text
+concept {
+    IfcRelDefinesByProperties:RelatingPropertyDefinition -> IfcPropertySetDefinitionSet
+}
+```

backend/apps/ifc_validation/checks/statistics/templates/Usage_of_bylayer_IfcCurveStyle.md

@@ -0,0 +1,13 @@
+# Usage of 'by layer' IfcCurveStyle
+
+> SCOPE IFC2X3+
+
+There has been a bug in the IfcOpenShell rule execution that caused spaces within string literals of express rules to get dropped. Therefore IfcCurveStyle_WR11 execution was wrong. This template uncovers whether this pattern is present in vendor-created uploads, informing us whether it is safe to update the validation service logic.
+
+```text
+concept {
+    IfcCurveStyle:CurveWidth -> IfcDescriptiveMeasure
+    IfcDescriptiveMeasure -> constraint_0
+    constraint_0[label="=by layer"]
+}
+```