Skip to content

Commit e552dc8

Browse files
PatrickCmdPatrickCmd
committed
OpenConceptLab/ocl_issues#22 | Retired concepts and/or mappings are now excluded from exports
1 parent 496b7d5 commit e552dc8

File tree

8 files changed

+161
-29
lines changed

8 files changed

+161
-29
lines changed

‎core/collections/views.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@
3131
CollectionVersionSummaryDetailSerializer, CollectionReferenceDetailSerializer)
3232
from core.collections.utils import is_version_specified
3333
from core.common.constants import (
34-
HEAD, RELEASED_PARAM, PROCESSING_PARAM, OK_MESSAGE, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
34+
HEAD, INCLUDE_RETIRED_PARAM, RELEASED_PARAM, PROCESSING_PARAM, OK_MESSAGE, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
3535
)
3636
from core.common.mixins import (
3737
ConceptDictionaryCreateMixin, ListWithHeadersMixin, ConceptDictionaryUpdateMixin,
@@ -594,7 +594,8 @@ class CollectionVersionExportView(ConceptContainerExportMixin, CollectionVersion
594594
def handle_export_version(self):
595595
version = self.get_object()
596596
try:
597-
export_collection.delay(version.id)
597+
include_retired = parse_boolean_query_param(self.request, INCLUDE_RETIRED_PARAM, "True")
598+
export_collection.delay(version.id, include_retired)
598599
return status.HTTP_202_ACCEPTED
599600
except AlreadyQueued:
600601
return status.HTTP_409_CONFLICT

‎core/common/mixins.py‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@
1414
from rest_framework.mixins import ListModelMixin, CreateModelMixin
1515
from rest_framework.response import Response
1616

17-
from core.common.constants import HEAD, ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW, ACCESS_TYPE_NONE, INCLUDE_FACETS, \
17+
from core.common.constants import HEAD, ACCESS_TYPE_EDIT, ACCESS_TYPE_VIEW, ACCESS_TYPE_NONE, INCLUDE_FACETS, INCLUDE_RETIRED_PARAM, \
1818
LIST_DEFAULT_LIMIT, HTTP_COMPRESS_HEADER, CSV_DEFAULT_LIMIT, FACETS_ONLY
1919
from core.common.permissions import HasPrivateAccess, HasOwnership, CanViewConceptDictionary
2020
from core.common.services import S3
21-
from .utils import write_csv_to_s3, get_csv_from_s3, get_query_params_from_url_string, compact_dict_by_values
21+
from .utils import write_csv_to_s3, get_csv_from_s3, get_query_params_from_url_string, compact_dict_by_values, parse_boolean_query_param
2222

2323
logger = logging.getLogger('oclapi')
2424

@@ -536,16 +536,8 @@ def get_object(self):
536536
return instance
537537

538538
def get(self, request, *args, **kwargs): # pylint: disable=unused-argument
539-
version = self.get_object()
540-
logger.debug(
541-
'Export requested for %s version %s - Requesting AWS-S3 key', self.entity.lower(), version.version
542-
)
543-
if version.is_head and not request.user.is_staff:
544-
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
545-
546-
if version.has_export():
547-
export_url = version.get_export_url()
548539

540+
def export_response():
549541
no_redirect = request.query_params.get('noRedirect', False) in ['true', 'True', True]
550542
if no_redirect:
551543
return Response(dict(url=export_url), status=status.HTTP_200_OK)
@@ -561,6 +553,24 @@ def get(self, request, *args, **kwargs): # pylint: disable=unused-argument
561553
response['Last-Updated-Timezone'] = settings.TIME_ZONE_PLACE
562554
return response
563555

556+
version = self.get_object()
557+
include_retired = parse_boolean_query_param(request, INCLUDE_RETIRED_PARAM, "True")
558+
logger.debug(
559+
'Export requested for %s version %s - Requesting AWS-S3 key', self.entity.lower(), version.version
560+
)
561+
562+
if version.is_head and not request.user.is_staff:
563+
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
564+
565+
if include_retired:
566+
if version.has_export():
567+
export_url = version.get_export_url()
568+
return export_response()
569+
else:
570+
if version.has_unretired_export():
571+
export_url = version.get_unretired_export_url()
572+
return export_response()
573+
564574
if version.is_exporting:
565575
return Response(status=status.HTTP_208_ALREADY_REPORTED)
566576

‎core/common/models.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -728,6 +728,11 @@ def is_exporting(self):
728728
def export_path(self):
729729
last_update = self.last_child_update.strftime('%Y%m%d%H%M%S')
730730
return self.generic_export_path(suffix="{}.zip".format(last_update))
731+
732+
@cached_property
733+
def exclude_retired_export_path(self):
734+
last_update = self.last_child_update.strftime('%Y%m%d%H%M%S')
735+
return self.generic_export_path(suffix="{}_unretired.zip".format(last_update))
731736

732737
def generic_export_path(self, suffix='*'):
733738
path = "{}/{}_{}.".format(self.parent_resource, self.mnemonic, self.version)
@@ -739,9 +744,15 @@ def generic_export_path(self, suffix='*'):
739744
def get_export_url(self):
740745
return S3.url_for(self.export_path)
741746

747+
def get_unretired_export_url(self):
748+
return S3.url_for(self.exclude_retired_export_path)
749+
742750
def has_export(self):
743751
return S3.exists(self.export_path)
744752

753+
def has_unretired_export(self):
754+
return S3.exists(self.exclude_retired_export_path)
755+
745756

746757
class CelerySignalProcessor(RealTimeSignalProcessor):
747758
def handle_save(self, sender, instance, **kwargs):

‎core/common/tasks.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ def delete_organization(org_id):
3737

3838

3939
@app.task(base=QueueOnce, bind=True)
40-
def export_source(self, version_id):
40+
def export_source(self, version_id, include_retired=True):
4141
from core.sources.models import Source
4242
logger.info('Finding source version...')
4343

@@ -52,14 +52,14 @@ def export_source(self, version_id):
5252
version.add_processing(self.request.id)
5353
try:
5454
logger.info('Found source version %s. Beginning export...', version.version)
55-
write_export_file(version, 'source', 'core.sources.serializers.SourceVersionExportSerializer', logger)
55+
write_export_file(version, include_retired, 'source', 'core.sources.serializers.SourceVersionExportSerializer', logger)
5656
logger.info('Export complete!')
5757
finally:
5858
version.remove_processing(self.request.id)
5959

6060

6161
@app.task(base=QueueOnce, bind=True)
62-
def export_collection(self, version_id):
62+
def export_collection(self, version_id, include_retired=True):
6363
from core.collections.models import Collection
6464
logger.info('Finding collection version...')
6565

@@ -75,7 +75,7 @@ def export_collection(self, version_id):
7575
try:
7676
logger.info('Found collection version %s. Beginning export...', version.version)
7777
write_export_file(
78-
version, 'collection', 'core.collections.serializers.CollectionVersionExportSerializer', logger
78+
version, include_retired, 'collection', 'core.collections.serializers.CollectionVersionExportSerializer', logger
7979
)
8080
logger.info('Export complete!')
8181
finally:

‎core/common/utils.py‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,7 @@ def get_class(kls):
191191

192192

193193
def write_export_file(
194-
version, resource_type, resource_serializer_type, logger
194+
version, include_retired, resource_type, resource_serializer_type, logger
195195
): # pylint: disable=too-many-statements,too-many-locals,too-many-branches
196196
from core.concepts.models import Concept
197197
from core.mappings.models import Mapping
@@ -207,6 +207,12 @@ def write_export_file(
207207
logger.info('Done serializing attributes.')
208208

209209
batch_size = 100
210+
concepts_qs = version.concepts
211+
mappings_qs = version.mappings
212+
if not include_retired:
213+
concepts_qs = concepts_qs.filter(retired=False)
214+
mappings_qs = mappings_qs.filter(retired=False)
215+
210216
is_collection = resource_type == 'collection'
211217

212218
if is_collection:
@@ -329,7 +335,7 @@ def write_export_file(
329335
logger.info(file_path)
330336
logger.info('Done compressing. Uploading...')
331337

332-
s3_key = version.export_path
338+
s3_key = version.export_path if include_retired else version.exclude_retired_export_path
333339
S3.upload_file(
334340
key=s3_key, file_path=file_path, binary=True, metadata=dict(ContentType='application/zip'),
335341
headers={'content-type': 'application/zip'}

‎core/integration_tests/tests_collections.py‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -875,29 +875,29 @@ def test_post_303(self, s3_exists_mock):
875875
def test_post_202(self, s3_exists_mock, export_collection_mock):
876876
s3_exists_mock.return_value = False
877877
response = self.client.post(
878-
'/collections/coll/v1/export/',
878+
'/collections/coll/v1/export/?includeRetired=False',
879879
HTTP_AUTHORIZATION='Token ' + self.token,
880880
format='json'
881881
)
882882

883883
self.assertEqual(response.status_code, 202)
884884
s3_exists_mock.assert_called_once_with("username/coll_v1.{}.zip".format(self.v1_updated_at))
885-
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id)
885+
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id, False)
886886

887887
@patch('core.collections.views.export_collection')
888888
@patch('core.common.services.S3.exists')
889889
def test_post_409(self, s3_exists_mock, export_collection_mock):
890890
s3_exists_mock.return_value = False
891891
export_collection_mock.delay.side_effect = AlreadyQueued('already-queued')
892892
response = self.client.post(
893-
'/collections/coll/v1/export/',
893+
'/collections/coll/v1/export/?includeRetired=False',
894894
HTTP_AUTHORIZATION='Token ' + self.token,
895895
format='json'
896896
)
897897

898898
self.assertEqual(response.status_code, 409)
899899
s3_exists_mock.assert_called_once_with("username/coll_v1.{}.zip".format(self.v1_updated_at))
900-
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id)
900+
export_collection_mock.delay.assert_called_once_with(self.collection_v1.id, False)
901901

902902

903903
class CollectionVersionListViewTest(OCLAPITestCase):
@@ -1017,6 +1017,62 @@ def test_export_collection(self, s3_mock): # pylint: disable=too-many-locals
10171017
import shutil
10181018
shutil.rmtree(latest_temp_dir)
10191019

1020+
@patch('core.common.utils.S3')
1021+
def test_unretired_export_collection(self, s3_mock): # pylint: disable=too-many-locals
1022+
s3_mock.url_for = Mock(return_value='https://s3-url')
1023+
s3_mock.upload_file = Mock()
1024+
source = OrganizationSourceFactory()
1025+
concept1 = ConceptFactory(parent=source)
1026+
concept2 = ConceptFactory(parent=source)
1027+
mapping = MappingFactory(from_concept=concept2, to_concept=concept1, parent=source)
1028+
collection = OrganizationCollectionFactory()
1029+
collection.add_references([concept1.uri, concept2.uri, mapping.uri])
1030+
collection.refresh_from_db()
1031+
1032+
export_collection(collection.id, include_retired=False) # pylint: disable=no-value-for-parameter
1033+
1034+
latest_temp_dir = get_latest_dir_in_path('/tmp/')
1035+
zipped_file = zipfile.ZipFile(latest_temp_dir + '/export.zip')
1036+
exported_data = json.loads(zipped_file.read('export.json').decode('utf-8'))
1037+
1038+
self.assertEqual(
1039+
exported_data,
1040+
{**CollectionVersionExportSerializer(collection).data, 'concepts': ANY, 'mappings': ANY, 'references': ANY}
1041+
)
1042+
1043+
exported_concepts = exported_data['concepts']
1044+
expected_concepts = ConceptVersionExportSerializer(
1045+
[concept2.get_latest_version(), concept1.get_latest_version()], many=True
1046+
).data
1047+
1048+
self.assertEqual(len(exported_concepts), 2)
1049+
self.assertIn(expected_concepts[0], exported_concepts)
1050+
self.assertIn(expected_concepts[1], exported_concepts)
1051+
1052+
exported_mappings = exported_data['mappings']
1053+
expected_mappings = MappingDetailSerializer([mapping.get_latest_version()], many=True).data
1054+
1055+
self.assertEqual(len(exported_mappings), 1)
1056+
self.assertEqual(expected_mappings, exported_mappings)
1057+
1058+
exported_references = exported_data['references']
1059+
expected_references = CollectionReferenceSerializer(collection.references.all(), many=True).data
1060+
1061+
self.assertEqual(len(exported_references), 3)
1062+
self.assertIn(exported_references[0], expected_references)
1063+
self.assertIn(exported_references[1], expected_references)
1064+
self.assertIn(exported_references[2], expected_references)
1065+
1066+
s3_upload_key = collection.exclude_retired_export_path
1067+
s3_mock.upload_file.assert_called_once_with(
1068+
key=s3_upload_key, file_path=latest_temp_dir + '/export.zip', binary=True,
1069+
metadata={'ContentType': 'application/zip'}, headers={'content-type': 'application/zip'}
1070+
)
1071+
s3_mock.url_for.assert_called_once_with(s3_upload_key)
1072+
1073+
import shutil
1074+
shutil.rmtree(latest_temp_dir)
1075+
10201076

10211077
class CollectionConceptsViewTest(OCLAPITestCase):
10221078
def setUp(self):

‎core/integration_tests/tests_sources.py‎

Lines changed: 51 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -705,29 +705,29 @@ def test_post_303(self, s3_exists_mock):
705705
def test_post_202(self, s3_exists_mock, export_source_mock):
706706
s3_exists_mock.return_value = False
707707
response = self.client.post(
708-
'/sources/source1/v1/export/',
708+
'/sources/source1/v1/export/?includeRetired=False',
709709
HTTP_AUTHORIZATION='Token ' + self.token,
710710
format='json'
711711
)
712712

713713
self.assertEqual(response.status_code, 202)
714714
s3_exists_mock.assert_called_once_with("username/source1_v1.{}.zip".format(self.v1_updated_at))
715-
export_source_mock.delay.assert_called_once_with(self.source_v1.id)
715+
export_source_mock.delay.assert_called_once_with(self.source_v1.id, False)
716716

717717
@patch('core.sources.views.export_source')
718718
@patch('core.common.services.S3.exists')
719719
def test_post_409(self, s3_exists_mock, export_source_mock):
720720
s3_exists_mock.return_value = False
721721
export_source_mock.delay.side_effect = AlreadyQueued('already-queued')
722722
response = self.client.post(
723-
'/sources/source1/v1/export/',
723+
'/sources/source1/v1/export/?includeRetired=False',
724724
HTTP_AUTHORIZATION='Token ' + self.token,
725725
format='json'
726726
)
727727

728728
self.assertEqual(response.status_code, 409)
729729
s3_exists_mock.assert_called_once_with("username/source1_v1.{}.zip".format(self.v1_updated_at))
730-
export_source_mock.delay.assert_called_once_with(self.source_v1.id)
730+
export_source_mock.delay.assert_called_once_with(self.source_v1.id, False)
731731

732732

733733
class ExportSourceTaskTest(OCLAPITestCase):
@@ -778,6 +778,53 @@ def test_export_source(self, s3_mock): # pylint: disable=too-many-locals
778778
import shutil
779779
shutil.rmtree(latest_temp_dir)
780780

781+
@patch('core.common.utils.S3')
782+
def test_unretired_export_source(self, s3_mock): # pylint: disable=too-many-locals
783+
s3_mock.url_for = Mock(return_value='https://s3-url')
784+
s3_mock.upload_file = Mock()
785+
source = OrganizationSourceFactory()
786+
concept1 = ConceptFactory(parent=source)
787+
concept2 = ConceptFactory(parent=source)
788+
mapping = MappingFactory(from_concept=concept2, to_concept=concept1, parent=source)
789+
790+
source_v1 = OrganizationSourceFactory(mnemonic=source.mnemonic, organization=source.organization, version='v1')
791+
concept1.sources.add(source_v1)
792+
concept2.sources.add(source_v1)
793+
mapping.sources.add(source_v1)
794+
795+
export_source(source_v1.id, include_retired=False) # pylint: disable=no-value-for-parameter
796+
797+
latest_temp_dir = get_latest_dir_in_path('/tmp/')
798+
zipped_file = zipfile.ZipFile(latest_temp_dir + '/export.zip')
799+
exported_data = json.loads(zipped_file.read('export.json').decode('utf-8'))
800+
801+
self.assertEqual(
802+
exported_data, {**SourceVersionExportSerializer(source_v1).data, 'concepts': ANY, 'mappings': ANY}
803+
)
804+
805+
exported_concepts = exported_data['concepts']
806+
expected_concepts = ConceptVersionExportSerializer([concept2, concept1], many=True).data
807+
808+
self.assertEqual(len(exported_concepts), 2)
809+
self.assertIn(expected_concepts[0], exported_concepts)
810+
self.assertIn(expected_concepts[1], exported_concepts)
811+
812+
exported_mappings = exported_data['mappings']
813+
expected_mappings = MappingDetailSerializer([mapping], many=True).data
814+
815+
self.assertEqual(len(exported_mappings), 1)
816+
self.assertEqual(expected_mappings, exported_mappings)
817+
818+
s3_upload_key = source_v1.exclude_retired_export_path
819+
s3_mock.upload_file.assert_called_once_with(
820+
key=s3_upload_key, file_path=latest_temp_dir + '/export.zip', binary=True,
821+
metadata={'ContentType': 'application/zip'}, headers={'content-type': 'application/zip'}
822+
)
823+
s3_mock.url_for.assert_called_once_with(s3_upload_key)
824+
825+
import shutil
826+
shutil.rmtree(latest_temp_dir)
827+
781828

782829
class SourceLogoViewTest(OCLAPITestCase):
783830
def setUp(self):

‎core/sources/views.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from rest_framework.response import Response
1515

1616
from core.client_configs.views import ResourceClientConfigsView
17-
from core.common.constants import HEAD, RELEASED_PARAM, PROCESSING_PARAM, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
17+
from core.common.constants import HEAD, INCLUDE_RETIRED_PARAM, RELEASED_PARAM, PROCESSING_PARAM, NOT_FOUND, MUST_SPECIFY_EXTRA_PARAM_IN_BODY
1818
from core.common.mixins import ListWithHeadersMixin, ConceptDictionaryCreateMixin, ConceptDictionaryUpdateMixin, \
1919
ConceptContainerExportMixin, ConceptContainerProcessingMixin
2020
from core.common.permissions import CanViewConceptDictionary, CanEditConceptDictionary, HasAccessToVersionedObject, \
@@ -365,7 +365,8 @@ class SourceVersionExportView(ConceptContainerExportMixin, SourceVersionBaseView
365365
def handle_export_version(self):
366366
version = self.get_object()
367367
try:
368-
export_source.delay(version.id)
368+
include_retired = parse_boolean_query_param(self.request, INCLUDE_RETIRED_PARAM, "True")
369+
export_source.delay(version.id, include_retired)
369370
return status.HTTP_202_ACCEPTED
370371
except AlreadyQueued:
371372
return status.HTTP_409_CONFLICT

0 commit comments

Comments
 (0)