OpenConceptLab/ocl_issues#22 | Retired concepts and/or mappings are now excluded from exports

Author: PatrickCmd

core/collections/views.py

@@ -31,7 +31,7 @@
     CollectionVersionSummaryDetailSerializer, CollectionReferenceDetailSerializer)
 from core.collections.utils import is_version_specified
 from core.common.constants import (
-    HEAD, RELEASED_PARAM, PROCESSING_PARAM, OK_MESSAGE, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
+    HEAD, INCLUDE_RETIRED_PARAM, RELEASED_PARAM, PROCESSING_PARAM, OK_MESSAGE, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
 )
 from core.common.mixins import (
     ConceptDictionaryCreateMixin, ListWithHeadersMixin, ConceptDictionaryUpdateMixin,
@@ -594,7 +594,8 @@ class CollectionVersionExportView(ConceptContainerExportMixin, CollectionVersion
     def handle_export_version(self):
         version = self.get_object()
         try:
-            export_collection.delay(version.id)
+            include_retired = parse_boolean_query_param(self.request, INCLUDE_RETIRED_PARAM)
+            export_collection.delay(version.id, include_retired)
             return status.HTTP_202_ACCEPTED
         except AlreadyQueued:
             return status.HTTP_409_CONFLICT

core/common/models.py

@@ -722,6 +722,11 @@ def is_exporting(self):
     def export_path(self):
         last_update = self.last_child_update.strftime('%Y%m%d%H%M%S')
         return self.generic_export_path(suffix="{}.zip".format(last_update))
+    
+    @cached_property
+    def exclude_retired_export_path(self):
+        last_update = self.last_child_update.strftime('%Y%m%d%H%M%S')
+        return self.generic_export_path(suffix="{}_non_retired.zip".format(last_update))
 
     def generic_export_path(self, suffix='*'):
         path = "{}/{}_{}.".format(self.parent_resource, self.mnemonic, self.version)

core/common/tasks.py

@@ -43,7 +43,7 @@ def delete_organization(org_id):
 
 
 @app.task(base=QueueOnce, bind=True)
-def export_source(self, version_id):
+def export_source(self, version_id, include_retired=True):
     from core.sources.models import Source
     logger.info('Finding source version...')
 
@@ -58,14 +58,14 @@ def export_source(self, version_id):
     version.add_processing(self.request.id)
     try:
         logger.info('Found source version %s.  Beginning export...', version.version)
-        write_export_file(version, 'source', 'core.sources.serializers.SourceVersionExportSerializer', logger)
+        write_export_file(version, include_retired, 'source', 'core.sources.serializers.SourceVersionExportSerializer', logger)
         logger.info('Export complete!')
     finally:
         version.remove_processing(self.request.id)
 
 
 @app.task(base=QueueOnce, bind=True)
-def export_collection(self, version_id):
+def export_collection(self, version_id, include_retired=True):
     from core.collections.models import Collection
     logger.info('Finding collection version...')
 
@@ -81,7 +81,7 @@ def export_collection(self, version_id):
     try:
         logger.info('Found collection version %s.  Beginning export...', version.version)
         write_export_file(
-            version, 'collection', 'core.collections.serializers.CollectionVersionExportSerializer', logger
+            version, include_retired, 'collection', 'core.collections.serializers.CollectionVersionExportSerializer', logger
         )
         logger.info('Export complete!')
     finally:

core/common/utils.py

@@ -191,7 +191,7 @@ def get_class(kls):
 
 
 def write_export_file(
-        version, resource_type, resource_serializer_type, logger
+        version, include_retired, resource_type, resource_serializer_type, logger
 ):  # pylint: disable=too-many-statements,too-many-locals,too-many-branches
     from core.concepts.models import Concept
     from core.mappings.models import Mapping
@@ -207,6 +207,12 @@ def write_export_file(
     logger.info('Done serializing attributes.')
 
     batch_size = 100
+    concepts_qs = version.concepts
+    mappings_qs = version.mappings
+    if not include_retired:
+        concepts_qs = concepts_qs.filter(retired=False)
+        mappings_qs = mappings_qs.filter(retired=False)
+    
     is_collection = resource_type == 'collection'
 
     if is_collection:
@@ -330,6 +336,8 @@ def write_export_file(
     logger.info('Done compressing.  Uploading...')
 
     s3_key = version.export_path
+    if not include_retired:
+        s3_key = version.exclude_retired_export_path
     S3.upload_file(
         key=s3_key, file_path=file_path, binary=True, metadata=dict(ContentType='application/zip'),
         headers={'content-type': 'application/zip'}

core/integration_tests/tests_collections.py

@@ -875,29 +875,29 @@ def test_post_303(self, s3_exists_mock):
     def test_post_202(self, s3_exists_mock, export_collection_mock):
         s3_exists_mock.return_value = False
         response = self.client.post(
-            '/collections/coll/v1/export/',
+            '/collections/coll/v1/export/?includeRetired=False',
             HTTP_AUTHORIZATION='Token ' + self.token,
             format='json'
         )
 
         self.assertEqual(response.status_code, 202)
         s3_exists_mock.assert_called_once_with("username/coll_v1.{}.zip".format(self.v1_updated_at))
-        export_collection_mock.delay.assert_called_once_with(self.collection_v1.id)
+        export_collection_mock.delay.assert_called_once_with(self.collection_v1.id, False)
 
     @patch('core.collections.views.export_collection')
     @patch('core.common.services.S3.exists')
     def test_post_409(self, s3_exists_mock, export_collection_mock):
         s3_exists_mock.return_value = False
         export_collection_mock.delay.side_effect = AlreadyQueued('already-queued')
         response = self.client.post(
-            '/collections/coll/v1/export/',
+            '/collections/coll/v1/export/?includeRetired=False',
             HTTP_AUTHORIZATION='Token ' + self.token,
             format='json'
         )
 
         self.assertEqual(response.status_code, 409)
         s3_exists_mock.assert_called_once_with("username/coll_v1.{}.zip".format(self.v1_updated_at))
-        export_collection_mock.delay.assert_called_once_with(self.collection_v1.id)
+        export_collection_mock.delay.assert_called_once_with(self.collection_v1.id, False)
 
 
 class CollectionVersionListViewTest(OCLAPITestCase):

core/integration_tests/tests_sources.py

@@ -705,29 +705,29 @@ def test_post_303(self, s3_exists_mock):
     def test_post_202(self, s3_exists_mock, export_source_mock):
         s3_exists_mock.return_value = False
         response = self.client.post(
-            '/sources/source1/v1/export/',
+            '/sources/source1/v1/export/?includeRetired=False',
             HTTP_AUTHORIZATION='Token ' + self.token,
             format='json'
         )
 
         self.assertEqual(response.status_code, 202)
         s3_exists_mock.assert_called_once_with("username/source1_v1.{}.zip".format(self.v1_updated_at))
-        export_source_mock.delay.assert_called_once_with(self.source_v1.id)
+        export_source_mock.delay.assert_called_once_with(self.source_v1.id, False)
 
     @patch('core.sources.views.export_source')
     @patch('core.common.services.S3.exists')
     def test_post_409(self, s3_exists_mock, export_source_mock):
         s3_exists_mock.return_value = False
         export_source_mock.delay.side_effect = AlreadyQueued('already-queued')
         response = self.client.post(
-            '/sources/source1/v1/export/',
+            '/sources/source1/v1/export/?includeRetired=False',
             HTTP_AUTHORIZATION='Token ' + self.token,
             format='json'
         )
 
         self.assertEqual(response.status_code, 409)
         s3_exists_mock.assert_called_once_with("username/source1_v1.{}.zip".format(self.v1_updated_at))
-        export_source_mock.delay.assert_called_once_with(self.source_v1.id)
+        export_source_mock.delay.assert_called_once_with(self.source_v1.id, False)
 
 
 class ExportSourceTaskTest(OCLAPITestCase):

core/sources/views.py

@@ -14,7 +14,7 @@
 from rest_framework.response import Response
 
 from core.client_configs.views import ResourceClientConfigsView
-from core.common.constants import HEAD, RELEASED_PARAM, PROCESSING_PARAM, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
+from core.common.constants import HEAD, INCLUDE_RETIRED_PARAM, RELEASED_PARAM, PROCESSING_PARAM, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
 from core.common.mixins import ListWithHeadersMixin, ConceptDictionaryCreateMixin, ConceptDictionaryUpdateMixin, \
     ConceptContainerExportMixin, ConceptContainerProcessingMixin
 from core.common.permissions import CanViewConceptDictionary, CanEditConceptDictionary, HasAccessToVersionedObject, \
@@ -365,7 +365,8 @@ class SourceVersionExportView(ConceptContainerExportMixin, SourceVersionBaseView
     def handle_export_version(self):
         version = self.get_object()
         try:
-            export_source.delay(version.id)
+            include_retired = parse_boolean_query_param(self.request, INCLUDE_RETIRED_PARAM)
+            export_source.delay(version.id, include_retired)
             return status.HTTP_202_ACCEPTED
         except AlreadyQueued:
             return status.HTTP_409_CONFLICT