From 127762c1b91f2a519c0cd5e7ec9f1c71cbd38602 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Mon, 6 Jun 2022 12:31:25 -0700 Subject: [PATCH 01/98] Added voterContactSave API for ignoring specific contacts. --- .../voter_contact_save_doc.py | 74 +++++++++++++++++++ apis_v1/urls.py | 4 + apis_v1/views/views_docs.py | 12 ++- apis_v1/views/views_voter.py | 62 ++++++++++++++++ templates/apis_v1/apis_index.html | 6 ++ voter/models.py | 60 +++++++++++++++ 6 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 apis_v1/documentation_source/voter_contact_save_doc.py diff --git a/apis_v1/documentation_source/voter_contact_save_doc.py b/apis_v1/documentation_source/voter_contact_save_doc.py new file mode 100644 index 000000000..94fe5e81e --- /dev/null +++ b/apis_v1/documentation_source/voter_contact_save_doc.py @@ -0,0 +1,74 @@ +# apis_v1/documentation_source/voter_contact_save_doc.py +# Brought to you by We Vote. Be good. +# -*- coding: UTF-8 -*- + + +def doc_template_values(url_root): + """ + Show documentation about voterContactSave + """ + required_query_parameter_list = [ + { + 'name': 'api_key', + 'value': 'string (from post, cookie, or get (in that order))', # boolean, integer, long, string + 'description': 'The unique key provided to any organization using the WeVoteServer APIs', + }, + { + 'name': 'voter_device_id', + 'value': 'string', # boolean, integer, long, string + 'description': 'An 88 character unique identifier linked to a voter record on the server. ', + }, + { + 'name': 'email_address_text', + 'value': 'string', # boolean, integer, long, string + 'description': 'The email of the VoterContactEmail that we want to alter.', + }, + ] + + optional_query_parameter_list = [ + { + 'name': 'ignore_voter_contact', + 'value': 'boolean', # boolean, integer, long, string + 'description': 'Set to true if the voter wants to ignore this VoterContact entry.', + }, + ] + + potential_status_codes_list = [ + { + 'code': 'VALID_VOTER_DEVICE_ID_MISSING', + 'description': 'A valid voter_device_id parameter was not included. Cannot proceed.', + }, + { + 'code': 'VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID', + 'description': 'No voter could be found from the voter_device_id', + }, + ] + + try_now_link_variables_dict = { + # 'voter_device_id': '', + } + + api_response = '{\n' \ + ' "success": boolean,\n' \ + ' "status": string,\n' \ + ' "ignore_voter_contact" boolean, \n' \ + ' "email_address_text": string, \n' \ + '}' + + template_values = { + 'api_name': 'voterContactSave', + 'api_slug': 'voterContactSave', + 'api_introduction': + 'Make changes to specific voterContactEmail entries.', + 'try_now_link': 'apis_v1:voterContactSaveView', + 'try_now_link_variables_dict': try_now_link_variables_dict, + 'url_root': url_root, + 'get_or_post': 'get', + 'required_query_parameter_list': required_query_parameter_list, + 'optional_query_parameter_list': optional_query_parameter_list, + 'api_response': api_response, + 'api_response_notes': + "", + 'potential_status_codes_list': potential_status_codes_list, + } + return template_values diff --git a/apis_v1/urls.py b/apis_v1/urls.py index 10db01df2..1b4530930 100644 --- a/apis_v1/urls.py +++ b/apis_v1/urls.py @@ -395,6 +395,8 @@ views_voter.voter_contact_list_retrieve_view, name='voterContactListRetrieveView'), re_path(r'^voterContactListSave/', views_voter.voter_contact_list_save_view, name='voterContactListSaveView'), + re_path(r'^voterContactSave/', views_voter.voter_contact_save_view, + name='voterContactSaveView'), re_path(r'^voterSMSPhoneNumberRetrieve/', views_voter.voter_sms_phone_number_retrieve_view, name='voterSMSPhoneNumberRetrieveView'), re_path(r'^voterSMSPhoneNumberSave/', views_voter.voter_sms_phone_number_save_view, @@ -755,6 +757,8 @@ name='voterContactListRetrieveDocs'), re_path(r'^docs/voterContactListSave/$', views_docs.voter_contact_list_save_doc_view, name='voterContactListSaveDocs'), + re_path(r'^docs/voterContactSave/$', views_docs.voter_contact_save_doc_view, + name='voterContactSaveDocs'), re_path(r'^docs/voterSMSPhoneNumberRetrieve/$', views_docs.voter_sms_phone_number_retrieve_doc_view, name='voterSMSPhoneNumberRetrieveDocs'), re_path(r'^docs/voterSMSPhoneNumberSave/$', diff --git a/apis_v1/views/views_docs.py b/apis_v1/views/views_docs.py index ebdfd3713..c8d6a3c4a 100644 --- a/apis_v1/views/views_docs.py +++ b/apis_v1/views/views_docs.py @@ -53,7 +53,7 @@ voter_ballot_items_retrieve_from_google_civic_doc, voter_ballot_list_retrieve_doc, \ voter_bookmark_off_save_doc, \ voter_bookmark_on_save_doc, voter_bookmark_status_retrieve_doc, \ - voter_contact_list_retrieve_doc, voter_contact_list_save_doc, \ + voter_contact_list_retrieve_doc, voter_contact_list_save_doc, voter_contact_save_doc, \ voter_count_doc, voter_create_doc, voter_email_address_retrieve_doc, voter_email_address_save_doc, \ voter_email_address_sign_in_doc, voter_email_address_verify_doc, voter_facebook_save_to_current_account_doc, \ voter_facebook_sign_in_retrieve_doc, voter_facebook_sign_in_save_doc, \ @@ -1557,6 +1557,16 @@ def voter_contact_list_save_doc_view(request): return render(request, 'apis_v1/api_doc_page.html', template_values) +def voter_contact_save_doc_view(request): + """ + Show documentation about voterContactSave + """ + url_root = WE_VOTE_SERVER_ROOT_URL + template_values = voter_contact_save_doc.doc_template_values(url_root) + template_values['voter_api_device_id'] = get_voter_api_device_id(request) + return render(request, 'apis_v1/api_doc_page.html', template_values) + + def voter_reaction_like_off_save_doc_view(request): """ Show documentation about voterReactionLikeOffSave diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index 463721b06..961b7b060 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -3072,6 +3072,7 @@ def voter_contact_list_save_view(request): # voterContactListSave status += augment_results['status'] # 2021-09-30 Requires Pro account which costs $90/month + # 2022-06-06 We are currently paying for this SendGrid account, so we can implement this # from email_outbound.controllers import augment_emails_for_voter_with_sendgrid # augment_results = augment_emails_for_voter_with_sendgrid(voter_we_vote_id=voter.we_vote_id) # status += augment_results['status'] @@ -3099,3 +3100,64 @@ def voter_contact_list_save_view(request): # voterContactListSave 'voter_contact_email_list_count': len(voter_contact_email_list), } return HttpResponse(json.dumps(json_data), content_type='application/json') + + +@csrf_exempt +def voter_contact_save_view(request): # voterContactSave + success = True + status = '' + email_ignored = False + + status, voter, voter_found, voter_device_link = views_voter_utils.get_voter_from_request(request, status) + email_address_text = request.GET.get('email_address_text', None) + ignore_voter_contact = positive_value_exists(request.GET.get('ignore_voter_contact', False)) + + try: + voter_we_vote_id = voter.we_vote_id + except Exception as e: + status += "VOTER_CONTACT_SAVE_NO_VOTER_WE_VOTE_ID: " + str(e) + " " + voter_we_vote_id = '' + + action_variable_found = positive_value_exists(ignore_voter_contact) + if not positive_value_exists(action_variable_found) or not positive_value_exists(email_address_text) or \ + not positive_value_exists(voter_we_vote_id): + email_ignored = False + status += "VOTER_CONTACT_SAVE_MISSING_KEY_VARIABLE " + success = False + json_data = { + 'status': status, + 'success': success, + 'email_address_text': email_address_text, + 'email_ignored': email_ignored, + } + return HttpResponse(json.dumps(json_data), content_type='application/json') + + voter_manager = VoterManager() + results = voter_manager.retrieve_voter_contact_email( + email_address_text=email_address_text, + imported_by_voter_we_vote_id=voter_we_vote_id, + read_only=False, + ) + if results['voter_contact_email_found']: + voter_contact_email = results['voter_contact_email'] + if ignore_voter_contact: + try: + voter_contact_email.ignore_contact = True + voter_contact_email.save() + email_ignored = True + except Exception as e: + success = False + status += "VOTER_CONTACT_EMAIL_FAILED_TO_SAVE_IGNORE: " + str(e) + ' ' + else: + status += "VOTER_CONTACT_EMAIL_ACTION_MISSING " + else: + status += "VOTER_CONTACT_EMAIL_NOT_FOUND " + json_data = { + 'status': status, + 'success': success, + 'action_variable_found': action_variable_found, + 'email_address_text': email_address_text, + 'email_ignored': email_ignored, + 'ignore_voter_contact': ignore_voter_contact, + } + return HttpResponse(json.dumps(json_data), content_type='application/json') diff --git a/templates/apis_v1/apis_index.html b/templates/apis_v1/apis_index.html index 968e3231a..4245cdbf0 100644 --- a/templates/apis_v1/apis_index.html +++ b/templates/apis_v1/apis_index.html @@ -293,6 +293,12 @@

Send the voter's contacts for processing. (Voter has explicitly opted in.) + + voterContactSave + + Make changes to specific voterContactEmail entries owned by this voter. + + {### ###} diff --git a/voter/models.py b/voter/models.py index ff595f047..51c8c6207 100644 --- a/voter/models.py +++ b/voter/models.py @@ -925,6 +925,66 @@ def retrieve_contact_email_augmented_list( } return results + def retrieve_voter_contact_email( + self, + email_address_text='', + imported_by_voter_we_vote_id='', + read_only=False): + success = True + status = "" + voter_contact_email = None + voter_contact_email_found = False + + if not positive_value_exists(email_address_text) or \ + not positive_value_exists(imported_by_voter_we_vote_id): + status += "MISSING_IMPORTED_BY_VOTER_WE_VOTE_ID " + results = { + 'success': False, + 'status': status, + 'voter_contact_email': voter_contact_email, + 'voter_contact_email_found': voter_contact_email_found, + } + return results + + try: + if positive_value_exists(read_only): + query = VoterContactEmail.objects.using('readonly').all() + else: + query = VoterContactEmail.objects.all() + query = query.filter(imported_by_voter_we_vote_id=imported_by_voter_we_vote_id) + query = query.filter(email_address_text__iexact=email_address_text) + list_of_voter_contact_emails = list(query) + if len(list_of_voter_contact_emails) > 1: + first_saved = False + for one_voter_contact_email in list_of_voter_contact_emails: + if first_saved: + if positive_value_exists(read_only): + try: + voter_contact_email.delete() + except Exception as e: + status += "VOTER_CONTACT_EMAIL_DELETE-EXCEPTION: " + str(e) + ' ' + else: + voter_contact_email = one_voter_contact_email + voter_contact_email_found = True + elif len(list_of_voter_contact_emails) == 1: + voter_contact_email = list_of_voter_contact_emails[0] + voter_contact_email_found = True + else: + voter_contact_email_found = False + status += "VOTER_CONTACT_EMAIL_NOT_FOUND " + except Exception as e: + voter_contact_email_found = False + status += "VOTER_CONTACT_EMAIL_NOT_FOUND-EXCEPTION: " + str(e) + ' ' + success = False + + results = { + 'success': success, + 'status': status, + 'voter_contact_email': voter_contact_email, + 'voter_contact_email_found': voter_contact_email_found, + } + return results + def retrieve_voter_contact_email_list(self, imported_by_voter_we_vote_id='', read_only=True): success = True status = "" From 80447c2a9dc26c8ffd174c3d229e9bbd7a223f63 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Tue, 7 Jun 2022 08:29:26 -0700 Subject: [PATCH 02/98] Added some status text to debug why voterGuidesFromFriendsUpcomingRetrieve isn't returning some friends. --- apis_v1/views/views_voter_guide.py | 3 ++- position/controllers.py | 6 +++++- .../data_cleanup_organization_analysis.html | 5 +++-- voter_guide/controllers.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/apis_v1/views/views_voter_guide.py b/apis_v1/views/views_voter_guide.py index 211f957b5..fb0f9d383 100644 --- a/apis_v1/views/views_voter_guide.py +++ b/apis_v1/views/views_voter_guide.py @@ -322,7 +322,8 @@ def voter_guides_from_friends_upcoming_retrieve_view(request): # voterGuidesFro return HttpResponse(json.dumps(results), content_type='application/json') results = voter_guides_upcoming_retrieve_for_api( - google_civic_election_id_list=google_civic_election_id_list, friends_vs_public=FRIENDS_AND_PUBLIC, + google_civic_election_id_list=google_civic_election_id_list, + friends_vs_public=FRIENDS_AND_PUBLIC, voter_we_vote_id=voter_we_vote_id) status += results['status'] diff --git a/position/controllers.py b/position/controllers.py index 10964c8cd..26129b603 100644 --- a/position/controllers.py +++ b/position/controllers.py @@ -2306,6 +2306,7 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot 'status': 'VALID_VOTER_DEVICE_ID_MISSING', 'success': False, 'count': 0, + 'friends_vs_public': friends_vs_public, 'kind_of_ballot_item': "UNKNOWN", 'ballot_item_id': 0, 'position_list': position_list, @@ -2323,10 +2324,12 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot voter_we_vote_id = "" if not positive_value_exists(voter_id): position_list = [] + status += "VALID_VOTER_ID_MISSING " json_data = { - 'status': "VALID_VOTER_ID_MISSING ", + 'status': status, 'success': False, 'count': 0, + 'friends_vs_public': friends_vs_public, 'kind_of_ballot_item': "UNKNOWN", 'ballot_item_id': 0, 'position_list': position_list, @@ -2457,6 +2460,7 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot 'status': status, 'success': success, 'count': positions_count, + 'friends_vs_public': friends_vs_public, 'kind_of_ballot_item': kind_of_ballot_item, 'ballot_item_id': ballot_item_id, 'ballot_item_we_vote_id': ballot_item_we_vote_id, diff --git a/templates/admin_tools/data_cleanup_organization_analysis.html b/templates/admin_tools/data_cleanup_organization_analysis.html index 8bb67d38e..b56223c9c 100644 --- a/templates/admin_tools/data_cleanup_organization_analysis.html +++ b/templates/admin_tools/data_cleanup_organization_analysis.html @@ -28,8 +28,9 @@

Endorser Analysis: {{ organization.organization_name }}

{{ organization.organization_twitter_handle }} ({{ organization.twitter_followers_count }} Followers) - {% endif %} (Refresh Twitter Details - s)
+ {% endif %} ( + Refresh Twitter Details + )
organization.twitter_user_id: {{ organization.twitter_user_id }}
organization.twitter_name: {{ organization.twitter_name }}
organization.twitter_id_from_link_to_organization: {{ organization.twitter_id_from_link_to_organization }}
diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 771b3dabc..90f017bd6 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -4677,13 +4677,14 @@ def retrieve_voter_guides_from_friends( voter_guide_list = [] voter_guide_list_found = False if not positive_value_exists(maximum_number_to_retrieve): - maximum_number_to_retrieve = 30 + maximum_number_to_retrieve = 50 friend_manager = FriendManager() friend_results = friend_manager.retrieve_current_friends(voter_we_vote_id) organization_we_vote_ids_from_friends = [] if friend_results['current_friend_list_found']: current_friend_list = friend_results['current_friend_list'] + status += "CURRENT_FRIEND_LIST: " for one_friend in current_friend_list: if one_friend.viewer_voter_we_vote_id == voter_we_vote_id: # If viewer is the voter, the friend is the viewee @@ -4691,16 +4692,20 @@ def retrieve_voter_guides_from_friends( one_friend = heal_current_friend(one_friend) try: organization_we_vote_ids_from_friends.append(one_friend.viewee_organization_we_vote_id) + status += "(viewee: " + one_friend.viewee_voter_we_vote_id + \ + "/" + one_friend.viewee_organization_we_vote_id + ")" except Exception as e: - pass + status += "APPEND_PROBLEM1: " + str(e) + ' ' else: # If viewer is NOT the voter, the friend is the viewer if not positive_value_exists(one_friend.viewer_organization_we_vote_id): one_friend = heal_current_friend(one_friend) try: organization_we_vote_ids_from_friends.append(one_friend.viewer_organization_we_vote_id) + status += "(viewer: " + one_friend.viewer_voter_we_vote_id + \ + "/" + one_friend.viewer_organization_we_vote_id + ")" except Exception as e: - pass + status += "APPEND_PROBLEM2: " + str(e) + ' ' if len(organization_we_vote_ids_from_friends) == 0: success = True From 6ca1bb8d79855acba0032ecf8b280fe8540968c9 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Tue, 7 Jun 2022 12:23:13 -0700 Subject: [PATCH 03/98] Created update routines to fix problem where voterGuidesFromFriendsUpcomingRetrieve isn't returning some friends. --- config/urls.py | 1 + friend/controllers.py | 1 - friend/urls.py | 9 ++- friend/views_admin.py | 107 ++++++++++++++++++++++++++++++++ organization/views_admin.py | 9 ++- position/controllers.py | 3 +- templates/voter/voter_list.html | 67 ++++++++++++-------- voter/models.py | 2 + voter/views_admin.py | 5 ++ voter_guide/views_admin.py | 5 +- 10 files changed, 173 insertions(+), 36 deletions(-) diff --git a/config/urls.py b/config/urls.py index 26b7645d7..769a6a813 100644 --- a/config/urls.py +++ b/config/urls.py @@ -43,6 +43,7 @@ re_path(r'^elected_official/', include(('elected_official.urls', 'elected_official'), namespace="elected_official")), re_path(r'^electoral_district/', include(('electoral_district.urls', 'electoral_district'), namespace="electoral_district")), re_path(r'^follow/', include(('follow.urls', 'follow'), namespace="follow")), + re_path(r'^friend/', include(('friend.urls', 'friend'), namespace="friend")), re_path(r'^google_custom_search/', include(('google_custom_search.urls', 'google_custom_search'), namespace="google_custom_search")), re_path(r'^health/', views.health_view), # A simple health check to make sure the site is running re_path(r'^image/', include(('image.urls', 'image'), namespace="image")), diff --git a/friend/controllers.py b/friend/controllers.py index 566a0b16c..b36eb101b 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -2842,7 +2842,6 @@ def move_friends_to_another_voter( for from_friend_entry in from_friend_list_remaining: # Delete the remaining friendship values try: - # Leave this turned off until testing is finished from_friend_entry.delete() except Exception as e: status += "PROBLEM_DELETING_FRIEND " + str(e) + ' ' diff --git a/friend/urls.py b/friend/urls.py index 597bb3137..2be9c662e 100644 --- a/friend/urls.py +++ b/friend/urls.py @@ -8,9 +8,8 @@ urlpatterns = [ - url(r'^$', views_admin.batches_home_view, name='batches_home',), - # url(r'^batch_action_list/$', views_admin.batch_action_list_view, name='batch_action_list'), - # url(r'^batch_action_list_process/$', views_admin.batch_action_list_process_view, name='batch_action_list_process'), - # url(r'^batch_list/$', views_admin.batch_list_view, name='batch_list'), - # url(r'^batch_list_process/$', views_admin.batch_list_process_view, name='batch_list_process'), + url(r'^current_friends_data_healing/$', + views_admin.current_friends_data_healing_view, name='current_friends_data_healing'), + url(r'^refresh_voter_friend_count/$', + views_admin.refresh_voter_friend_count_view, name='refresh_voter_friend_count'), ] diff --git a/friend/views_admin.py b/friend/views_admin.py index 366b3c15e..112dd8cfc 100644 --- a/friend/views_admin.py +++ b/friend/views_admin.py @@ -2,7 +2,114 @@ # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- +from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.http import HttpResponseRedirect +from django.urls import reverse + +from .models import CurrentFriend, FriendManager import wevote_functions.admin +from admin_tools.views import redirect_to_sign_in_page +from wevote_functions.functions import positive_value_exists +from voter.models import Voter, voter_has_authority, VoterManager, voter_setup logger = wevote_functions.admin.get_logger(__name__) + +@login_required +def current_friends_data_healing_view(request): + status = "" + number_of_linked_org_updates = 0 + number_of_voters_missing_linked_org = 0 + voter_we_vote_id_list = [] + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + + try: + query = Voter.objects.all() + query = query.filter(friend_count__gt=0) + query = query.values_list('we_vote_id', flat=True).distinct() + voter_we_vote_id_list = list(query) + except Exception as e: + messages.add_message(request, messages.ERROR, + 'Could not retrieve any voters with current_friend > 0: ' + str(e)) + + if len(voter_we_vote_id_list): + voter_manager = VoterManager() + for voter_we_vote_id in voter_we_vote_id_list: + linked_organization_we_vote_id = voter_manager.fetch_linked_organization_we_vote_id_by_voter_we_vote_id( + voter_we_vote_id=voter_we_vote_id) + if positive_value_exists(linked_organization_we_vote_id): + viewer_updated = CurrentFriend.objects \ + .filter(viewer_voter_we_vote_id__iexact=voter_we_vote_id) \ + .exclude(viewer_organization_we_vote_id__iexact=linked_organization_we_vote_id) \ + .update(viewer_organization_we_vote_id=linked_organization_we_vote_id) + if positive_value_exists(viewer_updated): + number_of_linked_org_updates += viewer_updated + + viewee_updated = CurrentFriend.objects \ + .filter(viewee_voter_we_vote_id__iexact=voter_we_vote_id) \ + .exclude(viewee_organization_we_vote_id__iexact=linked_organization_we_vote_id) \ + .update(viewee_organization_we_vote_id=linked_organization_we_vote_id) + if positive_value_exists(viewee_updated): + number_of_linked_org_updates += viewee_updated + else: + number_of_voters_missing_linked_org += 1 + + if number_of_linked_org_updates: + messages.add_message(request, messages.INFO, + 'linked_organization_we_vote_id updates: ' + str(number_of_linked_org_updates)) + else: + messages.add_message(request, messages.INFO, + 'No linked_organization_we_vote_id updates. ') + + if number_of_voters_missing_linked_org: + messages.add_message(request, messages.ERROR, + 'number_of_voters_missing_linked_org: ' + str(number_of_voters_missing_linked_org)) + + return HttpResponseRedirect(reverse('voter:voter_list', args=())) + + +@login_required +def refresh_voter_friend_count_view(request): + """ + + :param request: + :return: + """ + status = "" + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + + query = CurrentFriend.objects.using('readonly').all() + query = query.values_list('viewer_voter_we_vote_id', flat=True).distinct() + viewer_voter_we_vote_id_list = list(query) + + query = CurrentFriend.objects.using('readonly').all() + query = query.values_list('viewee_voter_we_vote_id', flat=True).distinct() + viewee_voter_we_vote_id_list = list(query) + + voter_we_vote_id_list = list(set(viewer_voter_we_vote_id_list + viewee_voter_we_vote_id_list)) + + friend_manager = FriendManager() + voter_manager = VoterManager() + voters_with_friends = 0 + for one_voter_we_vote_id in voter_we_vote_id_list: + friends_count = friend_manager.fetch_current_friends_count(voter_we_vote_id=one_voter_we_vote_id) + if positive_value_exists(friends_count): + voters_with_friends += 1 + results = voter_manager.retrieve_voter_by_we_vote_id(voter_we_vote_id=one_voter_we_vote_id) + if results['voter_found']: + results['voter'].friend_count = friends_count + results['voter'].save() + + if voters_with_friends: + messages.add_message(request, messages.INFO, 'voters_with_friends: ' + str(voters_with_friends)) + + return HttpResponseRedirect(reverse('voter:voter_list', args=())) diff --git a/organization/views_admin.py b/organization/views_admin.py index 0f65232f6..b0961d6e8 100644 --- a/organization/views_admin.py +++ b/organization/views_admin.py @@ -461,7 +461,11 @@ def organization_list_view(request): else: organization_list_query = organization_list_query.order_by('organization_name') - if positive_value_exists(show_organizations_without_email): + # wv02org35759 + if positive_value_exists(organization_search): + # Do not limit search + pass + elif positive_value_exists(show_organizations_without_email): organization_list_query = organization_list_query.filter( Q(organization_email__isnull=True) | Q(organization_email__exact='') @@ -486,6 +490,9 @@ def organization_list_view(request): ) else: organization_list_query = organization_list_query.filter(organization_type__iexact=organization_type_filter) + elif positive_value_exists(organization_search): + # Do not remove individuals from search + pass else: # By default, don't show individuals organization_list_query = organization_list_query.exclude(organization_type__iexact=INDIVIDUAL) diff --git a/position/controllers.py b/position/controllers.py index 26129b603..dcb9be35c 100644 --- a/position/controllers.py +++ b/position/controllers.py @@ -2302,8 +2302,9 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot results = is_voter_device_id_valid(voter_device_id) if not results['success']: position_list = [] + status += 'VALID_VOTER_DEVICE_ID_MISSING ' json_data = { - 'status': 'VALID_VOTER_DEVICE_ID_MISSING', + 'status': status, 'success': False, 'count': 0, 'friends_vs_public': friends_vs_public, diff --git a/templates/voter/voter_list.html b/templates/voter/voter_list.html index 9466326f4..6febd9102 100644 --- a/templates/voter/voter_list.html +++ b/templates/voter/voter_list.html @@ -50,6 +50,9 @@

Voters - {{ voter_list_found_count|intcomma }} Found


+ +
+ @@ -59,7 +62,9 @@

Voters - {{ voter_list_found_count|intcomma }} Found


Cache images locally for all voters, -Process Maintenance Status Flags +Process Maintenance Status Flags, +Update friend count, +CurrentFriends data healing
{% csrf_token %} @@ -121,6 +126,13 @@

Voters - {{ voter_list_found_count|intcomma }} Found

{% if has_contributed %}checked{% endif %} /> Has Contributed + + + +
@@ -134,16 +146,13 @@

Voters - {{ voter_list_found_count|intcomma }} Found

  We Vote ID You + Friends Facebook Tw Email / Phone Opt In - Org we_vote_id Contributions - Admin - Partner Org - Political Data - Verified Volunteer + Roles Voter Images @@ -157,11 +166,24 @@

Voters - {{ voter_list_found_count|intcomma }} Found

{{ voter.we_vote_id }} +
+ {% if voter.linked_organization_we_vote_id %} + + {{ voter.linked_organization_we_vote_id|default_if_none:"" }} + + {% else %} + Linked org missing + {% endif %} {% if voter.id == voter_id_signed_in %}*ME*{% else %}{% endif %} + + + {% if voter.friend_count %}{{ voter.friend_count }}{% else %}{% endif %} + @@ -189,35 +211,21 @@

Voters - {{ voter_list_found_count|intcomma }} Found

{% if voter.is_opt_in_newsletter %}Yes{% else %}{% endif %} - - - {% if voter.linked_organization_we_vote_id %} - - {{ voter.linked_organization_we_vote_id|default_if_none:"" }} - - {% endif %} - {{ voter.amount_spent }} - - {% if voter.is_admin %}Yes{% else %}{% endif %} - {% if voter.is_analytics_admin %}Analytics Yes{% else %}{% endif %} - + + {% if voter.is_admin %}Admin, {% else %}{% endif %} + {% if voter.is_analytics_admin %}Analytics_Admin, {% else %}{% endif %} - - {% if voter.is_partner_organization %}Yes{% else %}{% endif %} - + {% if voter.is_partner_organization %}Partner, {% else %}{% endif %} - - {% if voter.is_political_data_manager %}Manager{% elif voter.is_political_data_viewer %}Viewer{% endif %} - + {% if voter.is_political_data_manager %}Data_Manager, + {% elif voter.is_political_data_viewer %}Data_Viewer, {% endif %} - - {% if voter.is_verified_volunteer %}Yes{% else %}{% endif %} + {% if voter.is_verified_volunteer %}Verified_Volunteer, {% else %}{% endif %} @@ -275,6 +283,11 @@

Voters - {{ voter_list_found_count|intcomma }} Found

this.form.submit(); }); }); + $(function() { + $('#has_friends_id').change(function() { + this.form.submit(); + }); + }); diff --git a/voter/models.py b/voter/models.py index 51c8c6207..0c3fdcc07 100644 --- a/voter/models.py +++ b/voter/models.py @@ -2983,6 +2983,8 @@ def __repr__(self): last_name = models.CharField(verbose_name='last name', max_length=255, null=True, blank=True) date_joined = models.DateTimeField(verbose_name='date joined', auto_now_add=True) date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) + # friend_count helps us quickly identify voters who have friends + friend_count = models.PositiveIntegerField(default=None, null=True) state_code_for_display = models.CharField(max_length=2, null=True, blank=True) state_code_for_display_hidden = models.BooleanField(default=False) state_code_for_display_updated = models.BooleanField(default=False) # Meant to be a transitory field during update diff --git a/voter/views_admin.py b/voter/views_admin.py index cf1b2ccd0..e6cb303e5 100644 --- a/voter/views_admin.py +++ b/voter/views_admin.py @@ -1097,6 +1097,7 @@ def voter_list_view(request): is_political_data_viewer = request.GET.get('is_political_data_viewer', '') is_verified_volunteer = request.GET.get('is_verified_volunteer', '') has_contributed = request.GET.get('has_contributed', '') + has_friends = request.GET.get('has_friends', '') voter_api_device_id = get_voter_api_device_id(request) # We look in the cookies for voter_api_device_id voter_manager = VoterManager() @@ -1190,6 +1191,9 @@ def voter_list_view(request): if positive_value_exists(has_contributed): payments = StripePayments.objects.all() voter_query = voter_query.filter(we_vote_id__in=Subquery(payments.values('voter_we_vote_id'))) + if positive_value_exists(has_friends): + voter_query = voter_query.filter(friend_count__gt=0) + voter_query = voter_query.order_by('-friend_count') voter_list_found_count = voter_query.count() @@ -1222,6 +1226,7 @@ def voter_list_view(request): 'is_political_data_viewer': is_political_data_viewer, 'is_verified_volunteer': is_verified_volunteer, 'has_contributed': has_contributed, + 'has_friends': has_friends, 'messages_on_stage': messages_on_stage, 'password_proposed': password_proposed, 'voter_list': modified_voter_list, diff --git a/voter_guide/views_admin.py b/voter_guide/views_admin.py index 3f0029032..65856ebb5 100644 --- a/voter_guide/views_admin.py +++ b/voter_guide/views_admin.py @@ -1974,7 +1974,10 @@ def voter_guide_list_view(request): else: voter_guide_query = voter_guide_query.filter(google_civic_election_id__in=google_civic_election_id_list) - if not positive_value_exists(show_individuals): + if positive_value_exists(voter_guide_search): + # Allow individuals to be found during voter guide search + pass + elif not positive_value_exists(show_individuals): voter_guide_query = voter_guide_query.exclude(voter_guide_owner_type__iexact=INDIVIDUAL) if positive_value_exists(voter_guide_search): From a04f7ba3782812591dd0f20a04a031c6eaea3246 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Tue, 7 Jun 2022 15:35:47 -0700 Subject: [PATCH 04/98] Fixed bug with positionListForBallotItemFromFriends. --- candidate/models.py | 6 +++--- position/models.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/candidate/models.py b/candidate/models.py index 8b1de8da8..be587350a 100644 --- a/candidate/models.py +++ b/candidate/models.py @@ -198,7 +198,7 @@ def retrieve_all_candidates_for_office(self, office_id=0, office_we_vote_id='', candidate_query = candidate_query.filter(we_vote_id__in=candidate_we_vote_id_list) candidate_query = candidate_query.exclude(do_not_display_on_ballot=True) candidate_query = candidate_query.order_by('-twitter_followers_count') - candidate_list = candidate_query + candidate_list = list(candidate_query) if len(candidate_list): candidate_list_found = True @@ -214,8 +214,8 @@ def retrieve_all_candidates_for_office(self, office_id=0, office_we_vote_id='', status += 'FAILED retrieve_all_candidates_for_office ' + str(e) + ' ' success = False - for one_candidate in candidate_list: - candidate_we_vote_id_list.append(one_candidate.we_vote_id) + # for one_candidate in candidate_list: + # candidate_we_vote_id_list.append(one_candidate.we_vote_id) results = { 'success': success, diff --git a/position/models.py b/position/models.py index b8aa21fc3..8166325a5 100644 --- a/position/models.py +++ b/position/models.py @@ -2250,6 +2250,7 @@ def retrieve_all_positions_for_contest_office(self, retrieve_public_positions=Tr candidate_we_vote_id_list = results['candidate_we_vote_id_list'] # Retrieve the support positions for this contest_office_id + position_list = [] position_list_found = False try: if retrieve_public_positions: @@ -2281,11 +2282,11 @@ def retrieve_all_positions_for_contest_office(self, retrieve_public_positions=Tr # for contest_office (like we do for candidate) because we don't have to deal with # PERCENT_RATING data with measures if friends_we_vote_id_list is not False: - # Find positions from friends. Look for we_vote_id case insensitive. + # Find positions from friends. Look for we_vote_id case-insensitive. we_vote_id_filter = Q() for we_vote_id in friends_we_vote_id_list: we_vote_id_filter |= Q(voter_we_vote_id__iexact=we_vote_id) - position_list_query = position_list_query.filter(we_vote_id_filter) + position_list_query = position_list_query.filter(we_vote_id_filter) # Limit to positions in the last x years - currently we are not limiting # position_list = position_list.filter(election_id=election_id) From 1ec2a81504697d2a113b77d55de6c51ba503d45f Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Thu, 9 Jun 2022 10:41:15 -0700 Subject: [PATCH 05/98] Fixed crashing bug with friend invitations. --- friend/controllers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/friend/controllers.py b/friend/controllers.py index b36eb101b..82d695c33 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -2275,7 +2275,7 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): "voter_twitter_handle": "", "voter_twitter_description": "", # To be implemented "voter_twitter_followers_count": 0, # To be implemented - "state_code_for_display": friend_voter.state_code_for_display, + "state_code_for_display": "", # To be implemented "voter_email_address": one_friend_invitation.recipient_voter_email, "invitation_status": one_friend_invitation.invitation_status, "invitation_table": one_friend_invitation.invitation_table, From c809db5189cc85bafae44a5e1b06d6a96ba65bf7 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Mon, 13 Jun 2022 16:47:51 -0700 Subject: [PATCH 06/98] Added full time (temporary) debug logging in views_apple (3) To debug Sign In With Apple --- apis_v1/views/views_apple.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index a24d75613..998824a8d 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -18,6 +18,7 @@ logger = wevote_functions.admin.get_logger(__name__) WE_VOTE_SERVER_ROOT_URL = get_environment_variable("WE_VOTE_SERVER_ROOT_URL") +DEBUG_LOGGING = True def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView @@ -50,8 +51,8 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView if results: user_code = results['subject_registered_claim'] email = results['email'] - # for a splunk trail... - # logger.error("splunkapple Not an error: Sign in with Apple iOS, decrypted jwt: " + user_code + " " + email) + if DEBUG_LOGGING: + logger.error("awsApple Not an error: Sign in with Apple iOS, (no email decrypted jwt: " + user_code + " " + email) voter_manager = VoterManager() voter_device_link_manager = VoterDeviceLinkManager() @@ -82,7 +83,8 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView 'previously_signed_in_voter_found': previously_signed_in_voter_found, 'previously_signed_in_voter_we_vote_id': previously_signed_in_voter_we_vote_id, } - # logger.error('splunkapple ' + status) + if DEBUG_LOGGING: + logger.error('awsApple (no voter_starting_process_we_vote_id): ' + status) return HttpResponse(json.dumps(json_data), content_type='application/json') results = sign_in_with_apple_for_api( @@ -106,7 +108,8 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView ) status += merge_results['status'] - # logger.error('splunkapple ' + status) + if DEBUG_LOGGING: + logger.error('awsApple (after apple_sign_in_save_merge_if_needed): ' + status) json_data = { 'status': status, @@ -220,6 +223,8 @@ def sign_in_with_apple_for_api( except Exception as e: success_siwa = False status += "ERROR_APPLE_USER_NOT_CREATED_OR_UPDATED: " + str(e) + ' ' + if DEBUG_LOGGING: + logger.error(status) handle_exception(e, logger=logger, exception_message=status) results = { @@ -249,7 +254,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect print("Method is", request.method) if not request.method == 'POST': - logger.error('splunkapple appleSignInOauthRedirectDestination WRONG Method: ' + request.method) + logger.error('awsApple appleSignInOauthRedirectDestination WRONG Method: ' + request.method) try: access_token = request.POST['id_token'] @@ -266,7 +271,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect # print('id_token renamed as access_token: ', access_token) if critical_variable_missing: - logger.error('splunkapple ' + status) + logger.error('awsApple ' + status) return HttpResponseRedirect('https://WeVote.US/applesigninprocess') if 'user' in request.POST: @@ -314,8 +319,8 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect if results: user_code = results['subject_registered_claim'] email = results['email'] - # for a splunk trail... - # logger.error("splunkapple Not an error: Sign in with Apple WebApp, decrypted jwt", results) + if DEBUG_LOGGING: + logger.error("awsApple Not an error: Sign in with Apple WebApp, decrypted jwt", results) # voter_we_vote_id = fetch_voter_we_vote_id_from_voter_device_link(voter_device_id) voter_manager = VoterManager() @@ -339,7 +344,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect if not positive_value_exists(voter_starting_process_we_vote_id): logger.error( - 'splunkapple did not receive a voter_we_vote_id from voter_device_id in sign_in_with_apple_oauth_redirect_view ') + 'awsApple did not receive a voter_we_vote_id from voter_device_id in sign_in_with_apple_oauth_redirect_view ') results = sign_in_with_apple_for_api( user_code=user_code, @@ -361,7 +366,8 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect voter_starting_process=voter_starting_process, ) status += merge_results['status'] - # logger.error('splunkapple ' + status) + if DEBUG_LOGGING: + logger.error('awsApple ' + status) return HttpResponseRedirect(return_url) From 2070a2dc450110413ff067515397e39f9c6c6170 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Tue, 14 Jun 2022 06:37:54 -0700 Subject: [PATCH 07/98] Added some debugging --- import_export_batches/controllers.py | 6 +++++- import_export_batches/views_admin.py | 10 ++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/import_export_batches/controllers.py b/import_export_batches/controllers.py index 4be215871..9024e2af7 100755 --- a/import_export_batches/controllers.py +++ b/import_export_batches/controllers.py @@ -63,7 +63,7 @@ def create_batch_row_actions( office_objects_dict={}): """ Cycle through all BatchRow entries for this batch_header_id and move the values we can find into - the BatchRowActionYYY table so we can review it before importing it + the BatchRowActionYYY table, so we can review it before importing it :param batch_header_id: :param batch_description: :param batch_row_id: @@ -254,6 +254,9 @@ def create_batch_row_actions( # batch_row_action_politician = results['batch_row_action_politician'] number_of_batch_actions_created += 1 success = True + status += "CREATE_BATCH_ROW_ACTION_POSITION-START: " + status += results['status'] + print_to_log(logger=logger, exception_message_optional=status) elif kind_of_batch == IMPORT_BALLOT_ITEM: results = create_batch_row_action_ballot_item( batch_description=batch_description, @@ -2755,6 +2758,7 @@ def create_batch_row_action_position(batch_description, batch_header_map, one_ba kind_of_action = IMPORT_CREATE else: kind_of_action = IMPORT_TO_BE_DETERMINED + print_to_log(logger=logger, exception_message_optional=status) try: batch_row_action_position.batch_set_id = batch_description.batch_set_id diff --git a/import_export_batches/views_admin.py b/import_export_batches/views_admin.py index 619102a3f..f8058fa1e 100755 --- a/import_export_batches/views_admin.py +++ b/import_export_batches/views_admin.py @@ -390,6 +390,7 @@ def batch_action_list_view(request): batch_set_list = [] polling_location_we_vote_id = "" + status = "" batch_header_id = convert_to_int(request.GET.get('batch_header_id', 0)) kind_of_batch = request.GET.get('kind_of_batch', '') @@ -563,6 +564,7 @@ def batch_action_list_view(request): modified_batch_row_list.append(one_batch_row) elif kind_of_batch == POSITION: existing_results = batch_manager.retrieve_batch_row_action_position(batch_header_id, one_batch_row.id) + status += existing_results['status'] if existing_results['batch_row_action_found']: one_batch_row.batch_row_action = existing_results['batch_row_action_position'] one_batch_row.kind_of_batch = POSITION @@ -599,8 +601,8 @@ def batch_action_list_view(request): if one_state[0].lower() in active_state_codes: filtered_state_list.append(one_state) - messages.add_message(request, messages.INFO, 'Batch Row Count: {batch_row_count}' - ''.format(batch_row_count=batch_row_count)) + messages.add_message(request, messages.INFO, 'Batch Row Count: {batch_row_count}, status: {status}' + ''.format(batch_row_count=batch_row_count, status=status)) messages_on_stage = get_messages(request) @@ -900,7 +902,7 @@ def batch_action_list_export_voters_view(request): @login_required def batch_action_list_analyze_process_view(request): """ - Create BatchRowActions for either all of the BatchRows for batch_header_id, or only one with batch_row_id + Create BatchRowActions for either all the BatchRows for batch_header_id, or only one with batch_row_id :param request: :return: """ @@ -3132,7 +3134,7 @@ def retrieve_ballots_for_polling_locations_api_v4_view(request): from lat/long, and then the ballot items. Ballotpedia API v4 Reach out to Ballotpedia and retrieve (for one election): 1) Polling locations (so we can use those addresses to retrieve a representative set of ballots) - 2) Cycle through a portion of those map points, enough that we are caching all of the possible ballot items + 2) Cycle through a portion of those map points, enough that we are caching all the possible ballot items :param request: :return: """ From 141c9b53cb1c704d27bc99fb645b9fd9669c26a8 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 14 Jun 2022 12:11:02 -0700 Subject: [PATCH 08/98] Sign In With Apple experiment 2 --- apis_v1/views/views_apple.py | 2 +- apple/AppleResolver.py | 8 +++++--- requirements.txt | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index 998824a8d..59172c996 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -224,7 +224,7 @@ def sign_in_with_apple_for_api( success_siwa = False status += "ERROR_APPLE_USER_NOT_CREATED_OR_UPDATED: " + str(e) + ' ' if DEBUG_LOGGING: - logger.error(status) + logger.error("awsApple: ", status) handle_exception(e, logger=logger, exception_message=status) results = { diff --git a/apple/AppleResolver.py b/apple/AppleResolver.py index c635feab2..d84e1f332 100644 --- a/apple/AppleResolver.py +++ b/apple/AppleResolver.py @@ -5,7 +5,7 @@ import wevote_functions.admin logger = wevote_functions.admin.get_logger(__name__) - +DEBUG_LOGGING = True class AppleResolver(object): # https://gist.github.com/davidhariri/b053787aabc9a8a9cc0893244e1549fe @@ -47,8 +47,10 @@ def authenticate(cls, access_token, client_id): verified_payload = jwt.decode(access_token, apple_public_key_as_string, audience=client_id, - algorithm=public_key_info['alg']) - print('AppleResolver verified_payload: ', verified_payload) + algorithm="RS256") + # algorithm=public_key_info['alg']) + if DEBUG_LOGGING: + logger.error('awsApple AppleResolver verified_payload: ', verified_payload) # sub: # The subject registered claim identifies the principal that is the subject of the identity token. Since diff --git a/requirements.txt b/requirements.txt index 0ddbd504f..f1321dae2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,8 +37,7 @@ pyasn1-modules==0.2.8 py3dns==3.2.1 pygeoip==0.3.2 pyjwkest==1.4.2 -# 4/21/22, needed to update for PyJWT==1.7.1 # We are stuck at 1.7.1 until twillio can handle a newer version (as of 6/15/21) -PyJWT==2.4.0 # We are stuck at 1.7.1 until twillio can handle a newer version (as of 6/15/21) +PyJWT==2.4.0 python-magic==0.4.24 # Requires "brew install libmagic" or "brew upgrade libmagic" or "pip install libmagic" python3-openid -e git+git://github.com/wevote/python3-openid.git@master#egg=python3-openid pytz==2021.1 From 0e84f713440de23192bc127abb85b63d28537281 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 14 Jun 2022 12:15:45 -0700 Subject: [PATCH 09/98] Sign In With Apple experiment 2 --- apis_v1/views/views_apple.py | 5 +++-- apple/AppleResolver.py | 1 + apple/models.py | 6 ++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index 59172c996..27bcbddb7 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -52,7 +52,8 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView user_code = results['subject_registered_claim'] email = results['email'] if DEBUG_LOGGING: - logger.error("awsApple Not an error: Sign in with Apple iOS, (no email decrypted jwt: " + user_code + " " + email) + logger.error("awsApple Not an error: Sign in with Apple iOS, (no email decrypted jwt: " + user_code + + " " + email) voter_manager = VoterManager() voter_device_link_manager = VoterDeviceLinkManager() @@ -344,7 +345,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect if not positive_value_exists(voter_starting_process_we_vote_id): logger.error( - 'awsApple did not receive a voter_we_vote_id from voter_device_id in sign_in_with_apple_oauth_redirect_view ') + 'awsApple didnt receive a voter_we_vote_id from voter_device_id in sign_in_with_apple_oauth_redirect_view') results = sign_in_with_apple_for_api( user_code=user_code, diff --git a/apple/AppleResolver.py b/apple/AppleResolver.py index d84e1f332..8dd72be20 100644 --- a/apple/AppleResolver.py +++ b/apple/AppleResolver.py @@ -7,6 +7,7 @@ logger = wevote_functions.admin.get_logger(__name__) DEBUG_LOGGING = True + class AppleResolver(object): # https://gist.github.com/davidhariri/b053787aabc9a8a9cc0893244e1549fe diff --git a/apple/models.py b/apple/models.py index ea684b66f..7e0935ba9 100644 --- a/apple/models.py +++ b/apple/models.py @@ -2,10 +2,7 @@ # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- -from datetime import date from django.db import models -from wevote_functions.functions import positive_value_exists -from exception.models import handle_exception, print_to_log import wevote_functions.admin logger = wevote_functions.admin.get_logger(__name__) @@ -21,6 +18,7 @@ class AppleUser(models.Model): # objects = None # voter_device_id = models.CharField(verbose_name='voter device id', # max_length=255, null=False, blank=False, unique=True, default='DELETE_ME') + objects = None voter_we_vote_id = models.CharField(verbose_name="we vote id for the Apple ID owner", max_length=255, unique=True) user_code = models.CharField(verbose_name="User's apple id code", max_length=255, null=False, unique=False) email = models.EmailField(verbose_name='apple email address', max_length=255, unique=False, @@ -32,7 +30,7 @@ class AppleUser(models.Model): last_name = models.CharField( verbose_name="User's last_name from Apple", max_length=255, null=True, blank=True, unique=False) - # The next three are for debugging/statistics are are not necessary for sign in + # The next three are for debugging/statistics are not necessary for sign in apple_platform = models.CharField( verbose_name="Platform of Apple Device", max_length=32, null=True, blank=True, unique=False) apple_os_version = models.CharField( From 88dcfa12b7938338942c27a9996923bf4c848951 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 14 Jun 2022 14:24:10 -0700 Subject: [PATCH 10/98] Sign In With Apple experiment 3 --- apis_v1/views/views_apple.py | 3 +++ config/base.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index 27bcbddb7..3f7fd6262 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -264,6 +264,9 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect critical_variable_missing = True try: state_dict = json.loads(request.POST['state']) + + logger.error('awsApple post params on redirect:' + json.dump(state_dict)) + voter_device_id = state_dict['voter_device_id'] return_url = state_dict['return_url'] except Exception as e: diff --git a/config/base.py b/config/base.py index bb7697f29..90286fd3b 100644 --- a/config/base.py +++ b/config/base.py @@ -9,6 +9,8 @@ import os import pathlib import re +import sys + from django.core.exceptions import ImproperlyConfigured from django.db import connection @@ -111,7 +113,14 @@ def get_git_merge_date(): pattern = '""gm' list_of_files = [fn for fn in glob.glob('./*/**') if not os.path.basename(fn).endswith(('/', '_')) and - re.search(r"/+.*?\..*?$", fn)] # exclude new directories + re.search(r"/+.*?\..*?$", fn)] # exclude directories entries + for fi in list_of_files: # TODO: Test code, remove asap, 6/14/22 + posix_filepath = pathlib.Path(fi) + stat_of_file = posix_filepath.stat() + ts = int(stat_of_file.st_atime) + tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') + err = '[startupSteve] files: ' + fi + ' --- ' + tm + '\n' + sys.stderr.write(err) latest_file_string = max(list_of_files, key=os.path.getctime) posix_filepath = pathlib.Path(latest_file_string) stat_of_file = posix_filepath.stat() From 250a75ee76ca388aa967d06dd476257711786be6 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 14 Jun 2022 15:09:04 -0700 Subject: [PATCH 11/98] Sign In With Apple experiment 4 --- apis_v1/views/views_apple.py | 2 +- config/base.py | 1 + templates/admin_tools/index.html | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index 3f7fd6262..e8d0e5ebd 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -265,7 +265,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect try: state_dict = json.loads(request.POST['state']) - logger.error('awsApple post params on redirect:' + json.dump(state_dict)) + logger.error('awsApple post params on redirect:' + json.dumps(state_dict)) voter_device_id = state_dict['voter_device_id'] return_url = state_dict['return_url'] diff --git a/config/base.py b/config/base.py index 90286fd3b..3f6fc168d 100644 --- a/config/base.py +++ b/config/base.py @@ -121,6 +121,7 @@ def get_git_merge_date(): tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') err = '[startupSteve] files: ' + fi + ' --- ' + tm + '\n' sys.stderr.write(err) + sys.stdout.write(err) latest_file_string = max(list_of_files, key=os.path.getctime) posix_filepath = pathlib.Path(latest_file_string) stat_of_file = posix_filepath.stat() diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index d0eade21e..136644f33 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -172,7 +172,7 @@

Technical Information

{{ postgres_version }} - Git date: + Git date: (experiment 4) {{git_merge_date}} From 2aeeb0be147c52a9a32fed79b810967b163aadd7 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 15 Jun 2022 11:47:37 -0700 Subject: [PATCH 12/98] Sign In With Apple experiment 5 --- apple/AppleResolver.py | 3 ++- config/base.py | 21 ++++++++++++--------- templates/admin_tools/index.html | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/apple/AppleResolver.py b/apple/AppleResolver.py index 8dd72be20..69999ebd9 100644 --- a/apple/AppleResolver.py +++ b/apple/AppleResolver.py @@ -48,7 +48,8 @@ def authenticate(cls, access_token, client_id): verified_payload = jwt.decode(access_token, apple_public_key_as_string, audience=client_id, - algorithm="RS256") + algorithms=["RS256"]) + # algorithm="RS256") # algorithm=public_key_info['alg']) if DEBUG_LOGGING: logger.error('awsApple AppleResolver verified_payload: ', verified_payload) diff --git a/config/base.py b/config/base.py index 3f6fc168d..9023722d6 100644 --- a/config/base.py +++ b/config/base.py @@ -114,18 +114,21 @@ def get_git_merge_date(): list_of_files = [fn for fn in glob.glob('./*/**') if not os.path.basename(fn).endswith(('/', '_')) and re.search(r"/+.*?\..*?$", fn)] # exclude directories entries - for fi in list_of_files: # TODO: Test code, remove asap, 6/14/22 - posix_filepath = pathlib.Path(fi) - stat_of_file = posix_filepath.stat() - ts = int(stat_of_file.st_atime) - tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') - err = '[startupSteve] files: ' + fi + ' --- ' + tm + '\n' - sys.stderr.write(err) - sys.stdout.write(err) + # for fi in list_of_files: # TODO: Test code, remove asap, 6/14/22 + # posix_filepath = pathlib.Path(fi) + # stat_of_file = posix_filepath.stat() + # ts = int(stat_of_file.st_ctime) + # tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') + # err = '[startupSteve] files: ' + fi + ' --- ' + tm + '\n' + # sys.stderr.write(err) + # sys.stdout.write(err) latest_file_string = max(list_of_files, key=os.path.getctime) posix_filepath = pathlib.Path(latest_file_string) stat_of_file = posix_filepath.stat() - git_merge_date = str(datetime.datetime.fromtimestamp(stat_of_file.st_mtime)).split('.', 1)[0] + git_merge_date = str(datetime.datetime.fromtimestamp(stat_of_file.st_atime)).split('.', 1)[0] + ' ' + git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_mtime)).split('.', 1)[0] + ' ' + git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_ctime)).split('.', 1)[0] + out_string = git_merge_date + ' (' + latest_file_string + ')' # print(out_string) return out_string diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index 136644f33..0213368c4 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -172,7 +172,7 @@

Technical Information

{{ postgres_version }} - Git date: (experiment 4) + Git date: (experiment 5)  {{git_merge_date}} From ebd12cea2e53ebb53bf5d18c25558f8e8091a137 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 16 Jun 2022 10:49:17 -0700 Subject: [PATCH 13/98] Sign In With Apple working, Experiment 6 for git compile date There was a (not brilliant) breaking change in PyJWT v2 --- apis_v1/views/views_apple.py | 6 +++--- apple/AppleResolver.py | 4 +--- config/base.py | 37 +++++++++++++++++++------------- templates/admin_tools/index.html | 8 +++++-- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index e8d0e5ebd..6fc3c3dde 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -18,7 +18,7 @@ logger = wevote_functions.admin.get_logger(__name__) WE_VOTE_SERVER_ROOT_URL = get_environment_variable("WE_VOTE_SERVER_ROOT_URL") -DEBUG_LOGGING = True +DEBUG_LOGGING = False def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView @@ -264,8 +264,8 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect critical_variable_missing = True try: state_dict = json.loads(request.POST['state']) - - logger.error('awsApple post params on redirect:' + json.dumps(state_dict)) + if DEBUG_LOGGING: + logger.error('awsApple post params on redirect:' + json.dumps(state_dict)) voter_device_id = state_dict['voter_device_id'] return_url = state_dict['return_url'] diff --git a/apple/AppleResolver.py b/apple/AppleResolver.py index 69999ebd9..e749e88cd 100644 --- a/apple/AppleResolver.py +++ b/apple/AppleResolver.py @@ -5,7 +5,7 @@ import wevote_functions.admin logger = wevote_functions.admin.get_logger(__name__) -DEBUG_LOGGING = True +DEBUG_LOGGING = False class AppleResolver(object): @@ -49,8 +49,6 @@ def authenticate(cls, access_token, client_id): verified_payload = jwt.decode(access_token, apple_public_key_as_string, audience=client_id, algorithms=["RS256"]) - # algorithm="RS256") - # algorithm=public_key_info['alg']) if DEBUG_LOGGING: logger.error('awsApple AppleResolver verified_payload: ', verified_payload) diff --git a/config/base.py b/config/base.py index 9023722d6..25c162b18 100644 --- a/config/base.py +++ b/config/base.py @@ -5,16 +5,13 @@ import datetime import glob import json -# import logging import os import pathlib import re -import sys from django.core.exceptions import ImproperlyConfigured from django.db import connection - # SECURITY WARNING: don't run with debug turned on in production! # Override in local.py for development DEBUG = False @@ -114,24 +111,34 @@ def get_git_merge_date(): list_of_files = [fn for fn in glob.glob('./*/**') if not os.path.basename(fn).endswith(('/', '_')) and re.search(r"/+.*?\..*?$", fn)] # exclude directories entries - # for fi in list_of_files: # TODO: Test code, remove asap, 6/14/22 - # posix_filepath = pathlib.Path(fi) - # stat_of_file = posix_filepath.stat() - # ts = int(stat_of_file.st_ctime) - # tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') - # err = '[startupSteve] files: ' + fi + ' --- ' + tm + '\n' - # sys.stderr.write(err) - # sys.stdout.write(err) latest_file_string = max(list_of_files, key=os.path.getctime) posix_filepath = pathlib.Path(latest_file_string) stat_of_file = posix_filepath.stat() git_merge_date = str(datetime.datetime.fromtimestamp(stat_of_file.st_atime)).split('.', 1)[0] + ' ' git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_mtime)).split('.', 1)[0] + ' ' - git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_ctime)).split('.', 1)[0] - - out_string = git_merge_date + ' (' + latest_file_string + ')' + git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_ctime)).split('.', 1)[0] # was mtime here before experiment + merge_date = [] + for fi in list_of_files: # TODO: Test code, remove asap, 6/14/22 + posix_filepath = pathlib.Path(fi) + stat_of_file = posix_filepath.stat() + ts = int(stat_of_file.st_ctime) + tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') + line = { + 'date': tm, + 'file': fi, + } + merge_date.append(line) + + # out_string = git_merge_date + ' (' + latest_file_string + ')' # print(out_string) - return out_string + line = { + 'date': git_merge_date, + 'file': latest_file_string, + } + merge_date2 = sorted(merge_date, key=lambda x: x['date'], reverse=True) + + merge_date2.insert(0, line) + return merge_date2 def get_postgres_version(): diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index 0213368c4..c1bce4e8a 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -172,8 +172,12 @@

Technical Information

{{ postgres_version }} - Git date: (experiment 5)  - {{git_merge_date}} + Git date: (experiment 6)  +{# {{git_merge_date}}#} + {% for line in git_merge_date %} +

{{ line.date }} --- {{ line.file }}

+ {% endfor %} +

From 1e6f0f5a5f23a58b21287071ecb33765fcf3b8e8 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Sun, 19 Jun 2022 08:19:12 -0700 Subject: [PATCH 14/98] Added tools to generate mutual friend data, and return summaries for the friend list displays. --- .../documentation_source/friend_list_doc.py | 10 +- .../friend_lists_all_doc.py | 3 +- friend/controllers.py | 499 ++++++++++++++++-- friend/models.py | 207 +++++++- friend/urls.py | 4 + friend/views_admin.py | 28 +- templates/voter/voter_edit.html | 4 + templates/voter/voter_list.html | 3 +- voter_guide/controllers.py | 2 +- 9 files changed, 700 insertions(+), 60 deletions(-) diff --git a/apis_v1/documentation_source/friend_list_doc.py b/apis_v1/documentation_source/friend_list_doc.py index c9d82791b..47dd4b74e 100644 --- a/apis_v1/documentation_source/friend_list_doc.py +++ b/apis_v1/documentation_source/friend_list_doc.py @@ -59,7 +59,7 @@ def friend_list_doc_template_values(url_root): ' "state_code": string,\n' \ ' "kind_of_list": string, \n' \ ' "friend_list": list\n' \ - ' [\n' \ + ' [{\n' \ ' "voter_we_vote_id": string,\n' \ ' "voter_display_name": string,\n' \ ' "voter_photo_url_large": string,\n' \ @@ -74,8 +74,14 @@ def friend_list_doc_template_values(url_root): ' "invitation_status": string,\n' \ ' "invitation_sent_to": string,\n' \ ' "positions_taken": number,\n' \ + ' "mutual_friend_count": number,\n' \ + ' "mutual_friend_preview_list": list\n' \ + ' [{\n' \ + ' "friend_display_name": string,\n' \ + ' "friend_photo_url_medium": string,\n' \ + ' }],\n' \ ' "mutual_friends": number,\n' \ - ' ],\n' \ + ' }],\n' \ '}' template_values = { diff --git a/apis_v1/documentation_source/friend_lists_all_doc.py b/apis_v1/documentation_source/friend_lists_all_doc.py index 746bf1c74..cb2f9da32 100644 --- a/apis_v1/documentation_source/friend_lists_all_doc.py +++ b/apis_v1/documentation_source/friend_lists_all_doc.py @@ -57,7 +57,8 @@ def friend_lists_all_doc_template_values(url_root): 'api_slug': 'friendListsAll', 'api_introduction': "Request information about a voter's friends, including invitations to become a friend, " - "a list of current friends, and friends you share in common with another voter. This API differs from friendList in that it " + "a list of current friends, and friends you share in common with another voter. " + "This API differs from friendList in that it " "returns six different lists at once.", 'try_now_link': 'apis_v1:friendListsAllView', 'url_root': url_root, diff --git a/friend/controllers.py b/friend/controllers.py index 82d695c33..41fcc64c7 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -1,11 +1,13 @@ # friend/controllers.py # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- -from .models import ACCEPTED, FriendInvitationVoterLink, FriendManager, CURRENT_FRIENDS, \ + +from django.db.models import F, Q +from django.utils.timezone import localtime, now +from .models import ACCEPTED, CurrentFriend, FriendInvitationVoterLink, FriendManager, CURRENT_FRIENDS, \ DELETE_INVITATION_EMAIL_SENT_BY_ME, DELETE_INVITATION_VOTER_SENT_BY_ME, FRIEND_INVITATIONS_PROCESSED, \ FRIEND_INVITATIONS_SENT_BY_ME, FRIEND_INVITATIONS_SENT_TO_ME, FRIEND_INVITATIONS_WAITING_FOR_VERIFICATION, \ - IGNORE_SUGGESTION, SUGGESTED_FRIEND_LIST, \ - UNFRIEND_CURRENT_FRIEND + IGNORE_SUGGESTION, MutualFriend, SuggestedFriend, SUGGESTED_FRIEND_LIST, UNFRIEND_CURRENT_FRIEND from config.base import get_environment_variable from email_outbound.controllers import schedule_email_with_email_outbound_description, schedule_verification_email from email_outbound.models import EmailAddress, EmailManager, FRIEND_ACCEPTED_INVITATION_TEMPLATE, \ @@ -126,7 +128,7 @@ def delete_friends_for_voter(voter_to_delete_we_vote_id): return results friend_manager = FriendManager() - from_friend_results = friend_manager.retrieve_current_friends(voter_to_delete_we_vote_id, read_only=False) + from_friend_results = friend_manager.retrieve_current_friend_list(voter_to_delete_we_vote_id, read_only=False) from_friend_list = from_friend_results['current_friend_list'] for from_friend_entry in from_friend_list: @@ -1263,7 +1265,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify success = False status += "friend_invitation_email_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " - # And finally, create an organization for this brand new signed-in voter so they can create public opinions + # And finally, create an organization for this brand new signed-in voter, so they can create public opinions organization_name = voter.get_full_name() organization_image = voter.voter_photo_url() organization_type = INDIVIDUAL @@ -1676,7 +1678,8 @@ def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we "recipient_name": recipient_name, "recipient_voter_email": recipient_voter_email, "see_all_friend_requests_url": web_app_root_url_verified + "/friends", - "confirm_friend_request_url": web_app_root_url_verified + "/more/network/key/" + invitation_secret_key, + "confirm_friend_request_url": + web_app_root_url_verified + "/more/network/key/" + invitation_secret_key, "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + recipient_email_subscription_secret_key, "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", @@ -1979,10 +1982,426 @@ def friend_lists_all_for_api(voter_device_id, # friendListsAll return results -def get_current_friends_list(status, voter): +def generate_mutual_friends_for_all_voters(): + total_mutual_friends_created_count = 0 + total_mutual_friends_updated_count = 0 + total_mutual_friends_update_suppressed_count = 0 + status = "" + success = False + + current_friend_queryset = CurrentFriend.objects.all() + current_friend_queryset = current_friend_queryset.order_by('-date_last_changed') + current_friend_list = list(current_friend_queryset) + # Loop through them CurrentFriend list + for one_current_friend in current_friend_list: + generate_results = generate_mutual_friends_for_current_friend(current_friend=one_current_friend) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + if positive_value_exists(mutual_friends_created_count): + total_mutual_friends_created_count += mutual_friends_created_count + if positive_value_exists(mutual_friends_updated_count): + total_mutual_friends_updated_count += mutual_friends_updated_count + if positive_value_exists(mutual_friends_update_suppressed_count): + total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + + suggested_friend_queryset = SuggestedFriend.objects.all() + suggested_friend_queryset = suggested_friend_queryset.order_by('-date_last_changed') + suggested_friend_list = list(suggested_friend_queryset) + # Loop through them SuggestedFriend list + for one_suggested_friend in suggested_friend_list: + generate_results = generate_mutual_friends_for_suggested_friend(suggested_friend=one_suggested_friend) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + if positive_value_exists(mutual_friends_created_count): + total_mutual_friends_created_count += mutual_friends_created_count + if positive_value_exists(mutual_friends_updated_count): + total_mutual_friends_updated_count += mutual_friends_updated_count + if positive_value_exists(mutual_friends_update_suppressed_count): + total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + + if positive_value_exists(total_mutual_friends_created_count): + status += "created: " + str(total_mutual_friends_created_count) + " " + if positive_value_exists(total_mutual_friends_updated_count): + status += "updated: " + str(total_mutual_friends_updated_count) + " " + if positive_value_exists(total_mutual_friends_update_suppressed_count): + status += "update_suppressed: " + str(total_mutual_friends_update_suppressed_count) + " " + + results = { + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_one_voter(voter_we_vote_id='', update_existing_data=False): + status = "" + success = True friend_manager = FriendManager() voter_manager = VoterManager() - position_metrics_manager = PositionMetricsManager() + # Retrieve list of all CurrentFriend entries connected to voter_we_vote_id + current_friend_list_results = friend_manager.retrieve_current_friend_list(voter_we_vote_id=voter_we_vote_id) + if not current_friend_list_results['success']: + status += current_friend_list_results['status'] + success = False + results = { + 'success': success, + 'status': status, + } + return results + + current_friend_list = current_friend_list_results['current_friend_list'] + + # Loop through them CurrentFriend list + for one_current_friend in current_friend_list: + generate_results = generate_mutual_friends_for_current_friend(current_friend=one_current_friend) + status += generate_results['status'] + + results = { + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_current_friend(current_friend=None, update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + voter_we_vote_id = current_friend.viewer_voter_we_vote_id + friend_voter_we_vote_id = current_friend.viewee_voter_we_vote_id + + # Retrieve the voter_we_vote_id's of all mutual friends, based on query of CurrentFriend table + mutual_voter_we_vote_id_list_from_current_friends = \ + friend_manager.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id=voter_we_vote_id, + friend_voter_we_vote_id=friend_voter_we_vote_id) + + # Update CurrentFriend with mutual_friend_count + mutual_friend_count = len(mutual_voter_we_vote_id_list_from_current_friends) + # if positive_value_exists(mutual_friend_count): + # status += "MUTUAL_FRIEND_COUNT: " + str(mutual_friend_count) + " " + + generate_results = generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + mutual_friends_voter_we_vote_id_list_from_current_friends=mutual_voter_we_vote_id_list_from_current_friends, + second_friend_voter_we_vote_id=friend_voter_we_vote_id, + update_existing_data=update_existing_data) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + + change_to_save = False + mutual_friend_count_change = False + if not positive_value_exists(mutual_friend_count) and current_friend.mutual_friend_count is None: + pass + elif current_friend.mutual_friend_count != mutual_friend_count: + current_friend.mutual_friend_count = mutual_friend_count + current_friend.mutual_friend_count_last_updated = localtime(now()).date() # We Vote uses Pacific Time + change_to_save = True + mutual_friend_count_change = True + + always_update_serialized = True + if always_update_serialized or mutual_friend_count_change or update_existing_data: + # Note that in the SuggestedFriend table, we store both "directions" of the suggested friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B + preview_results = generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + second_friend_voter_we_vote_id=friend_voter_we_vote_id) + status += preview_results['status'] + if preview_results['success']: + mutual_friend_preview_list_serialized = preview_results['mutual_friend_preview_list_serialized'] + if not positive_value_exists(mutual_friend_preview_list_serialized) and \ + current_friend.mutual_friend_preview_list_serialized is None: + pass + elif current_friend.mutual_friend_preview_list_serialized != mutual_friend_preview_list_serialized: + current_friend.mutual_friend_preview_list_serialized = mutual_friend_preview_list_serialized + current_friend.mutual_friend_preview_list_update_needed = False + change_to_save = True + else: + status += "FAILED_TO_GENERATE_PREVIEW_LIST_CURRENT_FRIEND " + + if change_to_save: + current_friend.save() + + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_suggested_friend(suggested_friend=None, update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + voter_we_vote_id = suggested_friend.viewer_voter_we_vote_id + friend_voter_we_vote_id = suggested_friend.viewee_voter_we_vote_id + + # Retrieve the voter_we_vote_id's of all mutual friends, based on query of CurrentFriend table + mutual_voter_we_vote_id_list_from_current_friends = \ + friend_manager.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id=voter_we_vote_id, + friend_voter_we_vote_id=friend_voter_we_vote_id) + + # Update SuggestedFriend with mutual_friend_count + mutual_friend_count = len(mutual_voter_we_vote_id_list_from_current_friends) + # if positive_value_exists(mutual_friend_count): + # status += "MUTUAL_FRIEND_COUNT: " + str(mutual_friend_count) + " " + + generate_results = generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + mutual_friends_voter_we_vote_id_list_from_current_friends=mutual_voter_we_vote_id_list_from_current_friends, + second_friend_voter_we_vote_id=friend_voter_we_vote_id, + update_existing_data=update_existing_data) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + + change_to_save = False + mutual_friend_count_change = False + if not positive_value_exists(mutual_friend_count) and suggested_friend.mutual_friend_count is None: + pass + elif suggested_friend.mutual_friend_count != mutual_friend_count: + suggested_friend.mutual_friend_count = mutual_friend_count + suggested_friend.mutual_friend_count_last_updated = localtime(now()).date() # We Vote uses Pacific Time + + change_to_save = True + mutual_friend_count_change = True + + always_update_serialized = True + if always_update_serialized or mutual_friend_count_change or update_existing_data: + # Note that in the SuggestedFriend table, we store both "directions" of the suggested friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B + preview_results = generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + second_friend_voter_we_vote_id=friend_voter_we_vote_id) + status += preview_results['status'] + if preview_results['success']: + mutual_friend_preview_list_serialized = preview_results['mutual_friend_preview_list_serialized'] + if not positive_value_exists(mutual_friend_preview_list_serialized) and \ + suggested_friend.mutual_friend_preview_list_serialized is None: + pass + elif suggested_friend.mutual_friend_preview_list_serialized != mutual_friend_preview_list_serialized: + suggested_friend.mutual_friend_preview_list_serialized = mutual_friend_preview_list_serialized + suggested_friend.mutual_friend_preview_list_update_needed = False + change_to_save = True + else: + status += "FAILED_TO_GENERATE_PREVIEW_LIST_SUGGESTED_FRIEND " + + if change_to_save: + suggested_friend.save() + + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id='', + mutual_friends_voter_we_vote_id_list_from_current_friends=[], + second_friend_voter_we_vote_id='', + update_existing_data=False): + friend_manager = FriendManager() + mutual_friends_created_count = 0 + mutual_friends_update_suppressed_count = 0 + mutual_friends_updated_count = 0 + status = "" + success = True + voter_manager = VoterManager() + + # Retrieve list of MutualFriend entries already existing for this CurrentFriend entry, + # so we can avoid recreating one that already exists. + # AND so we can remove entries which aren't needed any more. + mutual_friend_results = friend_manager.retrieve_mutual_friend_list( + first_friend_voter_we_vote_id=first_friend_voter_we_vote_id, + second_friend_voter_we_vote_id=second_friend_voter_we_vote_id, + read_only=False, # We may need to edit or delete these entries + ) + if not mutual_friend_results['success']: + status += "FAILED_RETRIEVE_MUTUAL_FRIEND_LIST: " + mutual_friend_results['status'] + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + existing_mutual_friend_list = mutual_friend_results['mutual_friend_list'] + + existing_mutual_friend_voter_we_vote_id_list = [] + existing_mutual_friend_dict = {} + # Retrieve the existing MutualFriend entries + for one_mutual_friend in existing_mutual_friend_list: + if positive_value_exists(one_mutual_friend.mutual_friend_voter_we_vote_id): + existing_mutual_friend_voter_we_vote_id_list.append(one_mutual_friend.mutual_friend_voter_we_vote_id) + existing_mutual_friend_dict[one_mutual_friend.mutual_friend_voter_we_vote_id] = one_mutual_friend + + # Loop through all the friends that first_friend_voter_we_vote_id and second_friend_voter_we_vote_id share in common + for one_mutual_friend_voter_we_vote_id in mutual_friends_voter_we_vote_id_list_from_current_friends: + if one_mutual_friend_voter_we_vote_id in existing_mutual_friend_voter_we_vote_id_list: + if positive_value_exists(update_existing_data): + # If we are updating existing data continue + pass + else: + # If not updating existing data, move onto the next mutual friend + mutual_friends_update_suppressed_count += 1 + continue + # Retrieve the voter who is the mutual friend, so we can get the name and profile image + voter_results = voter_manager.retrieve_voter_by_we_vote_id(one_mutual_friend_voter_we_vote_id) + if not voter_results['success'] or not voter_results['voter_found']: + # If we can't retrieve the voter data for this mutual friend for any reason, + # go on to the next mutual friend + status += "FAILED_RETRIEVING_MUTUAL_FRIEND_VOTER: " + voter_results['status'] + " " + continue + mutual_friend_display_name = None + mutual_friend_display_name_exists = False + we_vote_hosted_profile_image_url_medium = None + mutual_friend_profile_image_exists = False + + voter = voter_results['voter'] + if positive_value_exists(voter.get_full_name(real_name_only=True)): + mutual_friend_display_name = voter.get_full_name(real_name_only=True) + mutual_friend_display_name_exists = True + if positive_value_exists(voter.we_vote_hosted_profile_image_url_medium): + we_vote_hosted_profile_image_url_medium = voter.we_vote_hosted_profile_image_url_medium + mutual_friend_profile_image_exists = True + + viewer_to_mutual_friend_friend_count = friend_manager.fetch_mutual_friends_count_from_current_friends( + voter_we_vote_id=first_friend_voter_we_vote_id, + friend_voter_we_vote_id=one_mutual_friend_voter_we_vote_id, + ) + viewee_to_mutual_friend_friend_count = friend_manager.fetch_mutual_friends_count_from_current_friends( + voter_we_vote_id=second_friend_voter_we_vote_id, + friend_voter_we_vote_id=one_mutual_friend_voter_we_vote_id, + ) + + if one_mutual_friend_voter_we_vote_id in existing_mutual_friend_voter_we_vote_id_list: + # Update existing record + if update_existing_data: + mutual_friend = existing_mutual_friend_dict[one_mutual_friend_voter_we_vote_id] + mutual_friends_updated_count += 1 + else: + print("Will not update_existing_data") + else: + # Create new record + mutual_friend = MutualFriend.objects.create( + viewer_voter_we_vote_id=first_friend_voter_we_vote_id, + viewee_voter_we_vote_id=second_friend_voter_we_vote_id, + mutual_friend_voter_we_vote_id=one_mutual_friend_voter_we_vote_id, + mutual_friend_display_name=mutual_friend_display_name, + mutual_friend_display_name_exists=mutual_friend_display_name_exists, + mutual_friend_we_vote_hosted_profile_image_url_medium=we_vote_hosted_profile_image_url_medium, + mutual_friend_profile_image_exists=mutual_friend_profile_image_exists, + viewer_to_mutual_friend_friend_count=viewer_to_mutual_friend_friend_count, + viewee_to_mutual_friend_friend_count=viewee_to_mutual_friend_friend_count, + ) + mutual_friends_created_count += 1 + + # Now delete any MutualFriends in existing_mutual_friend_voter_we_vote_id_list + # which are not in mutual_friends_voter_we_vote_id_list_from_current_friends + mutual_friend_entry_voter_we_vote_id_list_to_delete = \ + list(set(existing_mutual_friend_voter_we_vote_id_list) - + set(mutual_friends_voter_we_vote_id_list_from_current_friends)) + for one_mutual_friend_voter_we_vote_id in mutual_friend_entry_voter_we_vote_id_list_to_delete: + mutual_friend = existing_mutual_friend_dict[one_mutual_friend_voter_we_vote_id] + if hasattr(mutual_friend, 'mutual_friend_voter_we_vote_id'): + mutual_friend.delete() + + # if positive_value_exists(mutual_friends_created_count): + # status += "created: " + str(mutual_friends_created_count) + " " + # if positive_value_exists(mutual_friends_updated_count): + # status += "updated: " + str(mutual_friends_updated_count) + " " + # if positive_value_exists(mutual_friends_update_suppressed_count): + # status += "update_suppressed: " + str(mutual_friends_update_suppressed_count) + " " + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id='', + second_friend_voter_we_vote_id=''): + status = "" + success = True + maximum_number_to_return = 5 + + # Note that in the MutualFriend table, we store both "directions" of the mutual friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B, regardless of which one is looking + queryset = MutualFriend.objects.using('readonly').all() + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=first_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=first_friend_voter_we_vote_id)) + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=second_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=second_friend_voter_we_vote_id)) + queryset = queryset.filter( + Q(mutual_friend_display_name_exists=True) | + Q(mutual_friend_profile_image_exists=True)) + queryset = queryset\ + .annotate(combined_friend_count=F('viewer_to_mutual_friend_friend_count') + + F('viewee_to_mutual_friend_friend_count'))\ + .order_by('-combined_friend_count') + mutual_friend_list = queryset[:maximum_number_to_return] + + mutual_friend_preview_list_found = False + mutual_friend_preview_list = [] + mutual_friend_preview_list_serialized = None + for one_mutual_friend in mutual_friend_list: + one_friend_dict = { + "friend_display_name": one_mutual_friend.mutual_friend_display_name, + "friend_photo_url_medium": one_mutual_friend.mutual_friend_we_vote_hosted_profile_image_url_medium, + } + mutual_friend_preview_list.append(one_friend_dict) + + if len(mutual_friend_preview_list) > 0: + mutual_friend_preview_list_found = True + mutual_friend_preview_list_serialized = json.dumps(mutual_friend_preview_list) + + results = { + 'mutual_friend_preview_list_found': mutual_friend_preview_list_found, + 'mutual_friend_preview_list_serialized': mutual_friend_preview_list_serialized, + 'success': success, + 'status': status, + } + return results + + +def get_current_friends_list(status, voter): + friend_manager = FriendManager() friend_list = [] status += "KIND_OF_LIST-CURRENT_FRIENDS " retrieve_current_friends_as_voters_results = friend_manager.retrieve_current_friends_as_voters( @@ -2017,15 +2436,16 @@ def get_current_friends_list(status, voter): # status += "VOTER_COULD_NOT_BE_HEALED " + heal_results['status'] # else: # status += "COULD_NOT_RETRIEVE_VOTER_THAT_CAN_BE_SAVED " + voter_results['status'] - mutual_friends = friend_manager.fetch_mutual_friends_count(voter.we_vote_id, friend_voter.we_vote_id) + # mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( + # voter.we_vote_id, friend_voter.we_vote_id) # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) one_friend = { "voter_we_vote_id": friend_voter.we_vote_id, "voter_date_last_changed": friend_voter.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), "voter_display_name": friend_voter.get_full_name(), "voter_photo_url_large": friend_voter.we_vote_hosted_profile_image_url_large - if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) - else friend_voter.voter_photo_url(), + if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) + else friend_voter.voter_photo_url(), 'voter_photo_url_medium': friend_voter.we_vote_hosted_profile_image_url_medium, 'voter_photo_url_tiny': friend_voter.we_vote_hosted_profile_image_url_tiny, "voter_email_address": friend_voter.email, @@ -2036,7 +2456,14 @@ def get_current_friends_list(status, voter): "state_code_for_display": friend_voter.state_code_for_display, "invitation_status": "", # Not used with CurrentFriends "invitation_sent_to": "", # Not used with CurrentFriends - "mutual_friends": mutual_friends, + "mutual_friend_count": friend_voter.mutual_friend_count + if hasattr(friend_voter, "mutual_friend_count") and + positive_value_exists(friend_voter.mutual_friend_count) else 0, + "mutual_friend_preview_list": friend_voter.mutual_friend_preview_list + if hasattr(friend_voter, "mutual_friend_preview_list") else [], + "mutual_friends": friend_voter.mutual_friend_count + if hasattr(friend_voter, "mutual_friend_count") and + positive_value_exists(friend_voter.mutual_friend_count) else 0, # "positions_taken": positions_taken, } friend_list.append(one_friend) @@ -2064,7 +2491,7 @@ def get_friend_invitations_processed(status, voter): recipient_voter_email = one_friend_invitation.recipient_voter_email \ if hasattr(one_friend_invitation, "recipient_voter_email") \ else "" - mutual_friends = friend_manager.fetch_mutual_friends_count( + mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( voter.we_vote_id, friend_voter.we_vote_id) # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) @@ -2073,13 +2500,13 @@ def get_friend_invitations_processed(status, voter): "voter_date_last_changed": friend_voter.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), "voter_display_name": friend_voter.get_full_name(), "voter_photo_url_large": friend_voter.we_vote_hosted_profile_image_url_large - if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) - else friend_voter.voter_photo_url(), + if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) + else friend_voter.voter_photo_url(), 'voter_photo_url_medium': friend_voter.we_vote_hosted_profile_image_url_medium, 'voter_photo_url_tiny': friend_voter.we_vote_hosted_profile_image_url_tiny, "voter_email_address": friend_voter.email, "voter_twitter_handle": friend_voter.twitter_screen_name, - "voter_twitter_description": "", # To be implemented + "voter_twitter_description": "", # To be implemented "voter_twitter_followers_count": 0, # To be implemented "linked_organization_we_vote_id": friend_voter.linked_organization_we_vote_id, "state_code_for_display": friend_voter.state_code_for_display, @@ -2127,7 +2554,7 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): sent_to_me_friend_list = results['voter_list'] # Augment the line with voter information for friend_voter in sent_to_me_friend_list: # This is the voter who sent the invitation to me - mutual_friends = friend_manager.fetch_mutual_friends_count( + mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( voter.we_vote_id, friend_voter.we_vote_id) # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) @@ -2238,7 +2665,7 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): for friend_voter in sent_by_me_friend_list: # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) - mutual_friends = friend_manager.fetch_mutual_friends_count( + mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( voter.we_vote_id, friend_voter.we_vote_id) one_friend = { "voter_we_vote_id": friend_voter.we_vote_id, @@ -2317,8 +2744,9 @@ def get_suggested_friends_list(status, voter): status += "SUGGESTED_FRIEND_VOTER_COULD_NOT_BE_HEALED " + heal_results['status'] else: status += "SUGGESTED-COULD_NOT_RETRIEVE_VOTER_THAT_CAN_BE_SAVED " + voter_results['status'] - mutual_friends = \ - friend_manager.fetch_mutual_friends_count(voter.we_vote_id, suggested_friend.we_vote_id) + # mutual_friends = \ + # friend_manager.fetch_mutual_friends_count_from_current_friends( + # voter.we_vote_id, suggested_friend.we_vote_id) # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(suggested_friend) one_friend = { @@ -2326,8 +2754,8 @@ def get_suggested_friends_list(status, voter): "voter_date_last_changed": suggested_friend.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), "voter_display_name": suggested_friend.get_full_name(), "voter_photo_url_large": suggested_friend.we_vote_hosted_profile_image_url_large - if positive_value_exists(suggested_friend.we_vote_hosted_profile_image_url_large) - else suggested_friend.voter_photo_url(), + if positive_value_exists(suggested_friend.we_vote_hosted_profile_image_url_large) + else suggested_friend.voter_photo_url(), 'voter_photo_url_medium': suggested_friend.we_vote_hosted_profile_image_url_medium, 'voter_photo_url_tiny': suggested_friend.we_vote_hosted_profile_image_url_tiny, "voter_email_address": suggested_friend.email, @@ -2338,7 +2766,14 @@ def get_suggested_friends_list(status, voter): "state_code_for_display": suggested_friend.state_code_for_display, "invitation_status": "", # Not used with SuggestedFriendList "invitation_sent_to": "", # Not used with SuggestedFriendList - "mutual_friends": mutual_friends, + "mutual_friend_count": suggested_friend.mutual_friend_count + if hasattr(suggested_friend, "mutual_friend_count") and + positive_value_exists(suggested_friend.mutual_friend_count) else 0, + "mutual_friend_preview_list": suggested_friend.mutual_friend_preview_list + if hasattr(suggested_friend, "mutual_friend_preview_list") else [], + "mutual_friends": suggested_friend.mutual_friend_count + if hasattr(suggested_friend, "mutual_friend_count") and + positive_value_exists(suggested_friend.mutual_friend_count) else 0, # "positions_taken": positions_taken, } friend_list.append(one_friend) @@ -2594,7 +3029,9 @@ def message_to_friend_send_for_api( # 'friend_invite_sent': True, # } # suggested_results = friend_manager.update_suggested_friend( - # voter_we_vote_id=sender_voter.we_vote_id, other_voter_we_vote_id=other_voter_we_vote_id, defaults=defaults) + # voter_we_vote_id=sender_voter.we_vote_id, + # other_voter_we_vote_id=other_voter_we_vote_id, + # defaults=defaults) # status += suggested_results['status'] results = { @@ -2649,7 +3086,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ friend_invitation_email_link_from_sender_list = \ friend_invitation_email_link_from_sender_results['friend_invitation_list'] - # TO SENDER: Now get existing invitations for the to_voter so we can see if an invitation already exists + # TO SENDER: Now get existing invitations for the to_voter, so we can see if an invitation already exists friend_invitation_email_link_to_sender_results = friend_manager.retrieve_friend_invitation_email_link_list( to_voter_we_vote_id) friend_invitation_email_link_to_sender_list = \ @@ -2688,7 +3125,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ friend_invitation_voter_link_from_sender_list = \ friend_invitation_voter_link_from_sender_results['friend_invitation_list'] - # TO SENDER: Now get existing invitations for the to_voter so we can see if an invitation already exists + # TO SENDER: Now get existing invitations for the to_voter, so we can see if an invitation already exists friend_invitation_voter_link_to_sender_results = friend_manager.retrieve_friend_invitation_voter_link_list( to_voter_we_vote_id) friend_invitation_voter_link_to_sender_list = \ @@ -2725,7 +3162,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ friend_invitation_voter_link_from_recipient_list = \ friend_invitation_voter_link_from_recipient_results['friend_invitation_list'] - # TO RECIPIENT: Now get existing invitations for the to_voter so we can see if an invitation already exists + # TO RECIPIENT: Now get existing invitations for the to_voter, so we can see if an invitation already exists friend_invitation_voter_link_to_recipient_results = friend_manager.retrieve_friend_invitation_voter_link_list( '', to_voter_we_vote_id) friend_invitation_voter_link_to_recipient_list = \ @@ -2806,9 +3243,9 @@ def move_friends_to_another_voter( voter_manager.fetch_linked_organization_we_vote_id_by_voter_we_vote_id(to_voter_we_vote_id) friend_manager = FriendManager() - from_friend_results = friend_manager.retrieve_current_friends(from_voter_we_vote_id, read_only=False) + from_friend_results = friend_manager.retrieve_current_friend_list(from_voter_we_vote_id, read_only=False) from_friend_list = from_friend_results['current_friend_list'] - to_friend_results = friend_manager.retrieve_current_friends(to_voter_we_vote_id, read_only=False) + to_friend_results = friend_manager.retrieve_current_friend_list(to_voter_we_vote_id, read_only=False) to_friend_list = to_friend_results['current_friend_list'] for from_friend_entry in from_friend_list: @@ -2837,7 +3274,9 @@ def move_friends_to_another_voter( friend_entries_not_moved += 1 status += "PROBLEM_UPDATING_FRIEND " + str(e) + ' ' - from_friend_list_remaining_results = friend_manager.retrieve_current_friends(from_voter_we_vote_id, read_only=False) + from_friend_list_remaining_results = friend_manager.retrieve_current_friend_list( + from_voter_we_vote_id, + read_only=False) from_friend_list_remaining = from_friend_list_remaining_results['current_friend_list'] for from_friend_entry in from_friend_list_remaining: # Delete the remaining friendship values diff --git a/friend/models.py b/friend/models.py index a715579c7..27fb0fe9e 100644 --- a/friend/models.py +++ b/friend/models.py @@ -60,6 +60,13 @@ class CurrentFriend(models.Model): verbose_name="voter we vote id person 2", max_length=255, null=True, blank=True, unique=False, db_index=True) viewee_organization_we_vote_id = models.CharField( max_length=255, null=True, blank=True, unique=False, db_index=True) + + mutual_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + mutual_friend_count_last_updated = models.DateTimeField(null=True) + + mutual_friend_preview_list_serialized = models.TextField(default=None, null=True) + mutual_friend_preview_list_update_needed = models.BooleanField(default=True) + date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) def fetch_other_organization_we_vote_id(self, one_we_vote_id): @@ -767,17 +774,20 @@ def fetch_current_friends_count(self, voter_we_vote_id): current_friends_count = 0 return current_friends_count - def fetch_mutual_friends_count(self, voter_we_vote_id, friend_we_vote_id): + def fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + self, + voter_we_vote_id='', + friend_voter_we_vote_id=''): """ TODO: This could be converted to database only calculation for better speed :param voter_we_vote_id: - :param friend_we_vote_id: + :param friend_voter_we_vote_id: :return: """ - mutual_friends_count = 0 + mutual_friends_voter_we_vote_id_list = [] - if not positive_value_exists(voter_we_vote_id) or not positive_value_exists(friend_we_vote_id): - return mutual_friends_count + if not positive_value_exists(voter_we_vote_id) or not positive_value_exists(friend_voter_we_vote_id): + return mutual_friends_voter_we_vote_id_list voter_friends_we_vote_id_list = [] friend_friends_we_vote_id_list = [] @@ -797,20 +807,29 @@ def fetch_mutual_friends_count(self, voter_we_vote_id, friend_we_vote_id): try: friend_friends_queryset = CurrentFriend.objects.using('readonly').all() friend_friends_queryset = friend_friends_queryset.filter( - Q(viewer_voter_we_vote_id__iexact=friend_we_vote_id) | - Q(viewee_voter_we_vote_id__iexact=friend_we_vote_id)) + Q(viewer_voter_we_vote_id__iexact=friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=friend_voter_we_vote_id)) friend_friends_list = list(friend_friends_queryset) for one_friend in friend_friends_list: - friend_friends_we_vote_id_list.append(one_friend.fetch_other_voter_we_vote_id(friend_we_vote_id)) + friend_friends_we_vote_id_list.append(one_friend.fetch_other_voter_we_vote_id(friend_voter_we_vote_id)) except Exception as e: - mutual_friends_count = 0 - return mutual_friends_count + mutual_friends_voter_we_vote_id_list = [] + return mutual_friends_voter_we_vote_id_list voter_set = set(voter_friends_we_vote_id_list) friend_set = set(friend_friends_we_vote_id_list) mutual_set = voter_set & friend_set - if mutual_set: - mutual_friends_count = len(mutual_set) + mutual_friends_voter_we_vote_id_list = list(mutual_set) + return mutual_friends_voter_we_vote_id_list + + def fetch_mutual_friends_count_from_current_friends(self, voter_we_vote_id='', friend_voter_we_vote_id=''): + mutual_friends_count = 0 + mutual_friends_voter_we_vote_id_list = \ + self.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id, friend_voter_we_vote_id) + + if mutual_friends_voter_we_vote_id_list: + mutual_friends_count = len(mutual_friends_voter_we_vote_id_list) return mutual_friends_count @@ -830,14 +849,14 @@ def fetch_suggested_friends_count(self, voter_we_vote_id): suggested_friends_count = 0 return suggested_friends_count - def retrieve_current_friends(self, voter_we_vote_id, read_only=True): + def retrieve_current_friend_list(self, voter_we_vote_id, read_only=True): status = "" current_friend_list = [] # The entries from CurrentFriend table current_friend_list_found = False if not positive_value_exists(voter_we_vote_id): success = False - status += 'VALID_VOTER_WE_VOTE_ID_MISSING' + status += 'VALID_VOTER_WE_VOTE_ID_MISSING ' results = { 'success': success, 'status': status, @@ -856,7 +875,7 @@ def retrieve_current_friends(self, voter_we_vote_id, read_only=True): Q(viewer_voter_we_vote_id__iexact=voter_we_vote_id) | Q(viewee_voter_we_vote_id__iexact=voter_we_vote_id)) current_friend_queryset = current_friend_queryset.order_by('-date_last_changed') - current_friend_list = current_friend_queryset + current_friend_list = list(current_friend_queryset) if len(current_friend_list): success = True @@ -875,7 +894,7 @@ def retrieve_current_friends(self, voter_we_vote_id, read_only=True): except Exception as e: success = False current_friend_list_found = False - status += 'FAILED retrieve_current_friends ' + str(e) + " " + status += 'FAILED retrieve_current_friend_list: ' + str(e) + " " current_friend_list = [] results = { @@ -942,17 +961,32 @@ def retrieve_current_friends_as_voters(self, voter_we_vote_id, read_only=True): status += 'AS_VOTERS_FAILED retrieve_current_friends_as_voters ' + str(e) + " " if current_friend_list_found: - current_friend_we_vote_id_list = [] + current_friend_dict = {} + current_friend_voter_we_vote_id_list = [] for current_friend_entry in current_friend_list: we_vote_id_of_friend = current_friend_entry.fetch_other_voter_we_vote_id(voter_we_vote_id) - current_friend_we_vote_id_list.append(we_vote_id_of_friend) + current_friend_voter_we_vote_id_list.append(we_vote_id_of_friend) + current_friend_dict[we_vote_id_of_friend] = current_friend_entry voter_manager = VoterManager() results = voter_manager.retrieve_voter_list_by_we_vote_id_list( - voter_we_vote_id_list=current_friend_we_vote_id_list, + voter_we_vote_id_list=current_friend_voter_we_vote_id_list, read_only=read_only) if results['voter_list_found']: - friend_list = results['voter_list'] + friend_list = [] + raw_friend_list = results['voter_list'] friend_list_found = True + # Augment friend_list with mutual_friend data + for one_voter in raw_friend_list: + if one_voter.we_vote_id in current_friend_dict: + current_friend = current_friend_dict[one_voter.we_vote_id] + one_voter.mutual_friend_count = current_friend.mutual_friend_count + if current_friend.mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(current_friend.mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + one_voter.mutual_friend_preview_list = mutual_friend_preview_list + friend_list.append(one_voter) results = { 'success': success, @@ -965,7 +999,7 @@ def retrieve_current_friends_as_voters(self, voter_we_vote_id, read_only=True): def retrieve_friends_we_vote_id_list(self, voter_we_vote_id): """ - This is similar to retrieve_current_friends, but only returns the we_vote_id + This is similar to retrieve_current_friend_list, but only returns the we_vote_id :param voter_we_vote_id: :return: """ @@ -2006,6 +2040,73 @@ def retrieve_friend_invitation_from_facebook(self, facebook_request_id, recipien } return results + def retrieve_mutual_friend_list( + self, + first_friend_voter_we_vote_id='', + second_friend_voter_we_vote_id='', + mutual_friend_voter_we_vote_id='', + read_only=True): + status = "" + success = True + mutual_friend_list = [] # The entries from MutualFriend table + mutual_friend_list_found = False + + if positive_value_exists(first_friend_voter_we_vote_id) and \ + positive_value_exists(second_friend_voter_we_vote_id): + pass + elif positive_value_exists(mutual_friend_voter_we_vote_id): + pass + else: + success = False + status += 'RETRIEVE_MUTUAL_FRIEND_LIST_MISSING_KEY_VARIABLE ' + results = { + 'success': success, + 'status': status, + 'mutual_friend_list_found': mutual_friend_list_found, + 'mutual_friend_list': mutual_friend_list, + } + return results + + try: + if positive_value_exists(read_only): + queryset = MutualFriend.objects.using('readonly').all() + else: + queryset = MutualFriend.objects.all() + if positive_value_exists(first_friend_voter_we_vote_id): + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=first_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=first_friend_voter_we_vote_id)) + if positive_value_exists(second_friend_voter_we_vote_id): + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=second_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=second_friend_voter_we_vote_id)) + if positive_value_exists(mutual_friend_voter_we_vote_id): + queryset = queryset.filter( + mutual_friend_voter_we_vote_id=mutual_friend_voter_we_vote_id, + ) + queryset = queryset.order_by('-date_last_changed') + mutual_friend_list = list(queryset) + + if len(mutual_friend_list): + mutual_friend_list_found = True + status += 'MUTUAL_FRIEND_LIST_RETRIEVED ' + else: + mutual_friend_list_found = False + status += 'NO_MUTUAL_FRIEND_LIST_RETRIEVED ' + except Exception as e: + success = False + mutual_friend_list_found = False + status += 'FAILED retrieve_mutual_friend_list: ' + str(e) + " " + mutual_friend_list = [] + + results = { + 'success': success, + 'status': status, + 'mutual_friend_list_found': mutual_friend_list_found, + 'mutual_friend_list': mutual_friend_list, + } + return results + def unfriend_current_friend(self, acting_voter_we_vote_id, other_voter_we_vote_id): # Retrieve the existing friendship status = "" @@ -2189,9 +2290,11 @@ def retrieve_suggested_friend_list_as_voters(self, voter_we_vote_id='', read_onl filtered_suggested_friend_list_we_vote_ids = {} if suggested_friend_list_found: + suggested_friend_dict = {} voter_manager = VoterManager() for suggested_friend_entry in suggested_friend_list: we_vote_id_of_friend = suggested_friend_entry.fetch_other_voter_we_vote_id(voter_we_vote_id) + suggested_friend_dict[we_vote_id_of_friend] = suggested_friend_entry if we_vote_id_of_friend in friends_we_vote_id_list: # If this person is already a friend, don't suggest as a friend @@ -2217,8 +2320,21 @@ def retrieve_suggested_friend_list_as_voters(self, voter_we_vote_id='', read_onl voter_we_vote_id_list=ordered_suggested_friend_list_we_vote_ids, read_only=read_only) if results['voter_list_found']: - friend_list = results['voter_list'] + friend_list = [] + raw_friend_list = results['voter_list'] friend_list_found = True + # Augment friend_list with mutual_friend data + for one_voter in raw_friend_list: + if one_voter.we_vote_id in suggested_friend_dict: + suggested_friend = suggested_friend_dict[one_voter.we_vote_id] + one_voter.mutual_friend_count = suggested_friend.mutual_friend_count + if suggested_friend.mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(suggested_friend.mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + one_voter.mutual_friend_preview_list = mutual_friend_preview_list + friend_list.append(one_voter) results = { 'success': success, @@ -2232,7 +2348,7 @@ def retrieve_suggested_friend_list_as_voters(self, voter_we_vote_id='', read_onl def update_suggested_friends_starting_with_one_voter(self, starting_voter_we_vote_id, read_only=False): """ Note that we default to "read_only=False" (that is, the live db) since usually we are doing this update - right after adding a friend, and we want the new friend to be returned in "retrieve_current_friends". + right after adding a friend, and we want the new friend to be returned in "retrieve_current_friend_list". We use the live database ("read_only=False") because we don't want to create a race condition with the replicated read_only not being caught up with the master fast enough (since a friend was just created above.) @@ -2241,7 +2357,7 @@ def update_suggested_friends_starting_with_one_voter(self, starting_voter_we_vot :param read_only: :return: """ - all_friends_one_person_results = self.retrieve_current_friends(starting_voter_we_vote_id, read_only=read_only) + all_friends_one_person_results = self.retrieve_current_friend_list(starting_voter_we_vote_id, read_only=read_only) suggested_friend_created_count = 0 if all_friends_one_person_results['current_friend_list_found']: current_friend_list = all_friends_one_person_results['current_friend_list'] @@ -2336,6 +2452,42 @@ def update_suggested_friend(self, voter_we_vote_id, other_voter_we_vote_id, return results +class MutualFriend(models.Model): + """ + This is considered a "cache" table, with all data being generated and aggregated to speed up other processes. + This table helps us generate and update the array stored in CurrentFriend.mutual_friend_preview_list_serialized. + The "direction" doesn't matter, although it usually indicates who initiated the first friend invitation. + """ + viewer_voter_we_vote_id = models.CharField( + verbose_name="voter we vote id person 1", max_length=255, null=True, blank=True, unique=False, db_index=True) + viewee_voter_we_vote_id = models.CharField( + verbose_name="voter we vote id person 2", max_length=255, null=True, blank=True, unique=False, db_index=True) + mutual_friend_voter_we_vote_id = models.CharField( + max_length=255, null=True, blank=True, unique=False, db_index=True) + + mutual_friend_display_name = models.CharField(max_length=255, null=True, blank=True) + mutual_friend_display_name_exists = models.BooleanField(default=False, db_index=True) + + mutual_friend_we_vote_hosted_profile_image_url_medium = models.TextField(blank=True, null=True) + mutual_friend_profile_image_exists = models.BooleanField(default=False, db_index=True) + + # The more friends the viewer and the mutual_friend share, the more important this mutual friend is + viewer_to_mutual_friend_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + # The more friends the viewee and the mutual_friend share, the more important this mutual friend is + viewee_to_mutual_friend_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + + date_last_changed = models.DateTimeField(null=True, auto_now=True) + + def fetch_other_voter_we_vote_id(self, one_we_vote_id): + if one_we_vote_id == self.viewer_voter_we_vote_id: + return self.viewee_voter_we_vote_id + elif one_we_vote_id == self.viewee_voter_we_vote_id: + return self.viewer_voter_we_vote_id + else: + # If the we_vote_id passed in wasn't found, don't return another we_vote_id + return "" + + class SuggestedFriend(models.Model): """ This table stores possible friend connections. @@ -2352,6 +2504,13 @@ class SuggestedFriend(models.Model): verbose_name="second voter to remove suggested friend", max_length=255, null=True, blank=True, unique=False) friend_invite_sent = models.BooleanField(default=False) current_friends = models.BooleanField(default=False) + + mutual_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + mutual_friend_count_last_updated = models.DateTimeField(null=True) + + mutual_friend_preview_list_serialized = models.TextField(default=None, null=True) + mutual_friend_preview_list_update_needed = models.BooleanField(default=True) + date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) def fetch_other_voter_we_vote_id(self, one_we_vote_id): diff --git a/friend/urls.py b/friend/urls.py index 2be9c662e..d2c807ec1 100644 --- a/friend/urls.py +++ b/friend/urls.py @@ -10,6 +10,10 @@ urlpatterns = [ url(r'^current_friends_data_healing/$', views_admin.current_friends_data_healing_view, name='current_friends_data_healing'), + url(r'^generate_mutual_friends_for_all_voters/$', + views_admin.generate_mutual_friends_for_all_voters_view, name='generate_mutual_friends_for_all_voters'), + url(r'^generate_mutual_friends_for_one_voter/$', + views_admin.generate_mutual_friends_for_one_voter_view, name='generate_mutual_friends_for_one_voter'), url(r'^refresh_voter_friend_count/$', views_admin.refresh_voter_friend_count_view, name='refresh_voter_friend_count'), ] diff --git a/friend/views_admin.py b/friend/views_admin.py index 112dd8cfc..a1370e506 100644 --- a/friend/views_admin.py +++ b/friend/views_admin.py @@ -7,10 +7,11 @@ from django.http import HttpResponseRedirect from django.urls import reverse +from .controllers import generate_mutual_friends_for_all_voters, generate_mutual_friends_for_one_voter from .models import CurrentFriend, FriendManager import wevote_functions.admin from admin_tools.views import redirect_to_sign_in_page -from wevote_functions.functions import positive_value_exists +from wevote_functions.functions import convert_to_int, positive_value_exists from voter.models import Voter, voter_has_authority, VoterManager, voter_setup logger = wevote_functions.admin.get_logger(__name__) @@ -73,6 +74,31 @@ def current_friends_data_healing_view(request): return HttpResponseRedirect(reverse('voter:voter_list', args=())) +@login_required +def generate_mutual_friends_for_all_voters_view(request): + status = "" + + results = generate_mutual_friends_for_all_voters() + status += results['status'] + messages.add_message(request, messages.INFO, 'status: ' + str(status)) + + return HttpResponseRedirect(reverse('voter:voter_list', args=())) + + +@login_required +def generate_mutual_friends_for_one_voter_view(request): + status = "" + voter_id = request.GET.get('voter_id', 0) + voter_id = convert_to_int(voter_id) + voter_we_vote_id = request.GET.get('voter_we_vote_id', '') + + results = generate_mutual_friends_for_one_voter(voter_we_vote_id=voter_we_vote_id) + status += results['status'] + messages.add_message(request, messages.INFO, 'status: ' + str(status)) + + return HttpResponseRedirect(reverse('voter:voter_edit', args=(voter_id,))) + + @login_required def refresh_voter_friend_count_view(request): """ diff --git a/templates/voter/voter_edit.html b/templates/voter/voter_edit.html index 68270d4c2..265eea07c 100644 --- a/templates/voter/voter_edit.html +++ b/templates/voter/voter_edit.html @@ -334,6 +334,10 @@

{% if voter %}Voter Edit: {{ voter.first_name|default_if_none:"" }} {{ voter

Remove Facebook Authentication (all devices and browsers)

+ +

+ Generate mutual friends for this voter +


{% endif %} diff --git a/templates/voter/voter_list.html b/templates/voter/voter_list.html index 6febd9102..03edf7b73 100644 --- a/templates/voter/voter_list.html +++ b/templates/voter/voter_list.html @@ -64,7 +64,8 @@

Voters - {{ voter_list_found_count|intcomma }} Found

Cache images locally for all voters, Process Maintenance Status Flags, Update friend count, -CurrentFriends data healing +CurrentFriends data healing, +Generate mutual friends
{% csrf_token %} diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 90f017bd6..e88a600de 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -4680,7 +4680,7 @@ def retrieve_voter_guides_from_friends( maximum_number_to_retrieve = 50 friend_manager = FriendManager() - friend_results = friend_manager.retrieve_current_friends(voter_we_vote_id) + friend_results = friend_manager.retrieve_current_friend_list(voter_we_vote_id) organization_we_vote_ids_from_friends = [] if friend_results['current_friend_list_found']: current_friend_list = friend_results['current_friend_list'] From 5aa546f1e8b96088858c6cb207eec6e94d0bd34e Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Sun, 19 Jun 2022 08:22:50 -0700 Subject: [PATCH 15/98] Added tools to generate mutual friend data, and return summaries for the friend list displays. Added security check, so update scripts can only be run by admins. --- friend/views_admin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/friend/views_admin.py b/friend/views_admin.py index a1370e506..d9e27c076 100644 --- a/friend/views_admin.py +++ b/friend/views_admin.py @@ -78,6 +78,11 @@ def current_friends_data_healing_view(request): def generate_mutual_friends_for_all_voters_view(request): status = "" + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + results = generate_mutual_friends_for_all_voters() status += results['status'] messages.add_message(request, messages.INFO, 'status: ' + str(status)) @@ -88,6 +93,12 @@ def generate_mutual_friends_for_all_voters_view(request): @login_required def generate_mutual_friends_for_one_voter_view(request): status = "" + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + voter_id = request.GET.get('voter_id', 0) voter_id = convert_to_int(voter_id) voter_we_vote_id = request.GET.get('voter_we_vote_id', '') From 7a58e7335f54b211b562874019244a15cbfab1b8 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Sun, 19 Jun 2022 12:31:06 -0700 Subject: [PATCH 16/98] Fixed crashing bug with generate_mutual_friends_for_one_voter. --- friend/controllers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/friend/controllers.py b/friend/controllers.py index 41fcc64c7..189813c80 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -2043,7 +2043,9 @@ def generate_mutual_friends_for_one_voter(voter_we_vote_id='', update_existing_d friend_manager = FriendManager() voter_manager = VoterManager() # Retrieve list of all CurrentFriend entries connected to voter_we_vote_id - current_friend_list_results = friend_manager.retrieve_current_friend_list(voter_we_vote_id=voter_we_vote_id) + current_friend_list_results = friend_manager.retrieve_current_friend_list( + voter_we_vote_id=voter_we_vote_id, + read_only=False) if not current_friend_list_results['success']: status += current_friend_list_results['status'] success = False From 82eaf1cb0e9de9f1f4cabc5e06680c2ce2576f55 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Mon, 20 Jun 2022 08:58:06 -0700 Subject: [PATCH 17/98] Figured out why Voter Guide Possibilities weren't being assigned to the person entering them. --- friend/controllers.py | 4 +- .../voter_guide_possibility_list.html | 2 + voter_guide/models.py | 38 +++++++++++-------- voter_guide/views_admin.py | 33 ++++++++++++++-- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/friend/controllers.py b/friend/controllers.py index 189813c80..09df8c238 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -2355,7 +2355,7 @@ def generate_mutual_friend_preview_list_serialized_for_two_voters( second_friend_voter_we_vote_id=''): status = "" success = True - maximum_number_to_return = 5 + maximum_number_of_mutual_friends_to_return = 8 # Note that in the MutualFriend table, we store both "directions" of the mutual friendship: # person A looking at person B, AND @@ -2377,7 +2377,7 @@ def generate_mutual_friend_preview_list_serialized_for_two_voters( .annotate(combined_friend_count=F('viewer_to_mutual_friend_friend_count') + F('viewee_to_mutual_friend_friend_count'))\ .order_by('-combined_friend_count') - mutual_friend_list = queryset[:maximum_number_to_return] + mutual_friend_list = queryset[:maximum_number_of_mutual_friends_to_return] mutual_friend_preview_list_found = False mutual_friend_preview_list = [] diff --git a/templates/voter_guide/voter_guide_possibility_list.html b/templates/voter_guide/voter_guide_possibility_list.html index 03ef0daab..e757f0c26 100644 --- a/templates/voter_guide/voter_guide_possibility_list.html +++ b/templates/voter_guide/voter_guide_possibility_list.html @@ -144,6 +144,7 @@

{{ filtered_by_title }}

{% for voter in political_data_managers_list %} {% endfor %} +     @@ -197,6 +198,7 @@

{{ filtered_by_title }}

+ diff --git a/voter_guide/models.py b/voter_guide/models.py index 6eb0f42d9..7329a5b51 100644 --- a/voter_guide/models.py +++ b/voter_guide/models.py @@ -2236,21 +2236,23 @@ def retrieve_voter_guide_possibility( } return results - def retrieve_voter_guide_possibility_list(self, - order_by='', - start_number=0, - end_number=25, - search_string='', - google_civic_election_id=0, - hide_from_active_review=False, - cannot_find_endorsements=False, - candidates_missing_from_we_vote=False, - capture_detailed_comments=False, - from_prior_election=False, - ignore_this_source=False, - show_prior_years=False, - assigned_to_voter_we_vote_id=False, - return_count_only=False): + def retrieve_voter_guide_possibility_list( + self, + order_by='', + start_number=0, + end_number=25, + search_string='', + google_civic_election_id=0, + hide_from_active_review=False, + cannot_find_endorsements=False, + candidates_missing_from_we_vote=False, + capture_detailed_comments=False, + from_prior_election=False, + ignore_this_source=False, + show_prior_years=False, + assigned_to_no_one=False, + assigned_to_voter_we_vote_id=False, + return_count_only=False): start_number = convert_to_int(start_number) end_number = convert_to_int(end_number) hide_from_active_review = positive_value_exists(hide_from_active_review) @@ -2273,7 +2275,11 @@ def retrieve_voter_guide_possibility_list(self, # Default to only showing this year now = datetime.now() voter_guide_query = voter_guide_query.filter(date_last_changed__year=now.year) - if positive_value_exists(assigned_to_voter_we_vote_id): + if positive_value_exists(assigned_to_no_one): + voter_guide_query = voter_guide_query.filter( + Q(assigned_to_voter_we_vote_id__isnull=True) | + Q(assigned_to_voter_we_vote_id="")) + elif positive_value_exists(assigned_to_voter_we_vote_id): voter_guide_query = voter_guide_query.filter( assigned_to_voter_we_vote_id__iexact=assigned_to_voter_we_vote_id) diff --git a/voter_guide/views_admin.py b/voter_guide/views_admin.py index 65856ebb5..d84ff3306 100644 --- a/voter_guide/views_admin.py +++ b/voter_guide/views_admin.py @@ -724,6 +724,7 @@ def voter_guide_create_process_view(request): organization_twitter_handle = extract_twitter_handle_from_text_string(organization_twitter_handle) voter_manager = VoterManager() + voter_who_submitted_is_political_data_manager = False organization_twitter_followers_count = 0 voter_who_submitted_name = "" voter_found = False @@ -732,6 +733,7 @@ def voter_guide_create_process_view(request): voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) if voter_results['voter_found']: voter = voter_results['voter'] + voter_who_submitted_is_political_data_manager = voter.is_political_data_manager voter_found = True voter_who_submitted_name = voter.get_full_name() voter_who_submitted_we_vote_id = voter.we_vote_id @@ -745,6 +747,7 @@ def voter_guide_create_process_view(request): voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) if voter_results['voter_found']: voter = voter_results['voter'] + voter_who_submitted_is_political_data_manager = voter.is_political_data_manager voter_found = True voter_who_submitted_name = voter.get_full_name() voter_who_submitted_we_vote_id = voter.we_vote_id @@ -758,6 +761,7 @@ def voter_guide_create_process_view(request): voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) if voter_results['voter_found']: voter = voter_results['voter'] + voter_who_submitted_is_political_data_manager = voter.is_political_data_manager voter_who_submitted_name = voter.get_full_name() voter_who_submitted_we_vote_id = voter.we_vote_id @@ -1258,6 +1262,10 @@ def voter_guide_create_process_view(request): 'voter_who_submitted_name': voter_who_submitted_name, 'voter_who_submitted_we_vote_id': voter_who_submitted_we_vote_id, } + if positive_value_exists(voter_who_submitted_we_vote_id) and voter_who_submitted_is_political_data_manager: + updated_values['assigned_to_name'] = voter_who_submitted_name + updated_values['assigned_to_voter_we_vote_id'] = voter_who_submitted_we_vote_id + if has_suggested_voter_guide_rights: updated_values['ignore_stored_positions'] = ignore_stored_positions updated_values['ignore_this_source'] = ignore_this_source @@ -2090,6 +2098,9 @@ def voter_guide_possibility_list_view(request): return redirect_to_sign_in_page(request, authority_required) assigned_to_voter_we_vote_id = request.GET.get('assigned_to_voter_we_vote_id', False) + assigned_to_no_one = positive_value_exists(request.GET.get('assigned_to_no_one', False)) + if positive_value_exists(assigned_to_no_one): + assigned_to_voter_we_vote_id = "ASSIGNED_TO_NO_ONE" from_prior_election = positive_value_exists(request.GET.get('from_prior_election', False)) google_civic_election_id = convert_to_int(request.GET.get('google_civic_election_id', 0)) show_all_elections = positive_value_exists(request.GET.get('show_all_elections', False)) @@ -2135,6 +2146,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, return_count_only=True) to_review_count = results['voter_guide_possibility_list_count'] @@ -2145,6 +2157,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, return_count_only=True) from_prior_election_count = results['voter_guide_possibility_list_count'] @@ -2155,6 +2168,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, return_count_only=True) cannot_find_endorsements_count = results['voter_guide_possibility_list_count'] @@ -2165,6 +2179,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, return_count_only=True) candidates_missing_count = results['voter_guide_possibility_list_count'] @@ -2175,6 +2190,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, return_count_only=True) capture_detailed_comments_count = results['voter_guide_possibility_list_count'] @@ -2195,6 +2211,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, from_prior_election=from_prior_election) if results['success']: @@ -2210,6 +2227,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, cannot_find_endorsements=cannot_find_endorsements) if results['success']: @@ -2225,6 +2243,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, candidates_missing_from_we_vote=candidates_missing_from_we_vote) if results['success']: @@ -2240,6 +2259,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, capture_detailed_comments=capture_detailed_comments) if results['success']: @@ -2256,6 +2276,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, hide_from_active_review=hide_from_active_review) if results['success']: @@ -2271,6 +2292,7 @@ def voter_guide_possibility_list_view(request): search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, show_prior_years=show_all_elections, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, ignore_this_source=ignore_this_source) print(f"show_ignore_this_source results {results}") @@ -2286,6 +2308,7 @@ def voter_guide_possibility_list_view(request): end_number=end_number, search_string=voter_guide_possibility_search, google_civic_election_id=google_civic_election_id, + assigned_to_no_one=assigned_to_no_one, assigned_to_voter_we_vote_id=assigned_to_voter_we_vote_id, show_prior_years=show_all_elections, ) @@ -2430,9 +2453,13 @@ def voter_guide_possibility_list_process_view(request): voter_guide_possibility_manager = VoterGuidePossibilityManager() if positive_value_exists(assigned_to_voter_we_vote_id): - # Show voter guide possibilities assigned to on person - url_variables += "&assigned_to_voter_we_vote_id=" + str(assigned_to_voter_we_vote_id) - elif positive_value_exists(reassign_to_voter_we_vote_id): + if assigned_to_voter_we_vote_id == 'ASSIGNED_TO_NO_ONE': + url_variables += "&assigned_to_no_one=" + str(True) + else: + # Show voter guide possibilities assigned to on person + url_variables += "&assigned_to_voter_we_vote_id=" + str(assigned_to_voter_we_vote_id) + + if positive_value_exists(reassign_to_voter_we_vote_id): voter_manager = VoterManager() results = voter_manager.retrieve_voter_by_we_vote_id(reassign_to_voter_we_vote_id) if results['voter_found']: From 85fe0741b2044fe33808981c93139723248a969b Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Thu, 23 Jun 2022 12:08:04 -0700 Subject: [PATCH 18/98] Generating mutual friends information for friend invitations. Preventing VoterContactEmails saved prior to sign in from being migrated if the account you sign into already has that entry. Splitting the VoterContactEmails augmentation step into a second API call, for speed. --- apis_v1/views/views_voter.py | 14 ++- friend/controllers.py | 223 +++++++++++++++++++++++++++++++--- friend/models.py | 7 ++ voter/controllers_contacts.py | 12 ++ 4 files changed, 236 insertions(+), 20 deletions(-) diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index 961b7b060..194517a92 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -2061,7 +2061,7 @@ def voter_update_view(request): # voterUpdate try: voter_device_id = get_voter_device_id(request) # We standardize how we take in the voter_device_id except RequestDataTooBig: - status += "RequestDataTooBig" + status += "RequestDataTooBig " json_data = { 'status': status, 'success': False, @@ -3056,6 +3056,7 @@ def voter_contact_list_save_view(request): # voterContactListSave status, voter, voter_found, voter_device_link = views_voter_utils.get_voter_from_request(request, status) contacts_string = request.POST.get('contacts', None) + augment_voter_contact_emails_with_location = request.POST.get('augment_voter_contact_emails_with_location', False) delete_all_voter_contact_emails = request.POST.get('delete_all_voter_contact_emails', False) google_api_key_type = request.POST.get('google_api_key_type', 'ballot') @@ -3082,9 +3083,10 @@ def voter_contact_list_save_view(request): # voterContactListSave # augment_results = augment_emails_for_voter_with_snovio(voter_we_vote_id=voter.we_vote_id) # status += augment_results['status'] - from import_export_open_people.controllers import augment_emails_for_voter_with_open_people - augment_results = augment_emails_for_voter_with_open_people(voter_we_vote_id=voter.we_vote_id) - status += augment_results['status'] + if positive_value_exists(augment_voter_contact_emails_with_location): + from import_export_open_people.controllers import augment_emails_for_voter_with_open_people + augment_results = augment_emails_for_voter_with_open_people(voter_we_vote_id=voter.we_vote_id) + status += augment_results['status'] retrieve_results = voter_contact_list_retrieve_for_api(voter_we_vote_id=voter.we_vote_id) voter_contact_email_list = retrieve_results['voter_contact_email_list'] @@ -3093,11 +3095,13 @@ def voter_contact_list_save_view(request): # voterContactListSave json_data = { 'status': status, 'success': success, - 'we_vote_id_for_google_contacts': voter.we_vote_id, + 'augment_voter_contact_emails_with_location': augment_voter_contact_emails_with_location, 'contacts_stored': contacts_stored, + 'delete_all_voter_contact_emails': delete_all_voter_contact_emails, 'voter_contact_email_google_count': voter_contact_email_google_count, 'voter_contact_email_list': voter_contact_email_list, 'voter_contact_email_list_count': len(voter_contact_email_list), + 'we_vote_id_for_google_contacts': voter.we_vote_id, } return HttpResponse(json.dumps(json_data), content_type='application/json') diff --git a/friend/controllers.py b/friend/controllers.py index 09df8c238..0edade095 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -223,8 +223,8 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ if not voter_results['voter_found']: error_results = { - 'status': "ACCEPTING_VOTER_NOT_FOUND ", - 'success': False, + 'status': "ACCEPTING_VOTER_NOT_FOUND ", + 'success': False, } return error_results @@ -233,8 +233,8 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ original_sender_voter_results = voter_manager.retrieve_voter_by_we_vote_id(original_sender_we_vote_id) if not original_sender_voter_results['voter_found']: error_results = { - 'status': "ORIGINAL_SENDER_VOTER_NOT_FOUND ", - 'success': False, + 'status': "ORIGINAL_SENDER_VOTER_NOT_FOUND ", + 'success': False, } return error_results @@ -333,8 +333,8 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ status += send_results['status'] results = { - 'success': True, - 'status': status, + 'success': True, + 'status': status, } return results @@ -496,6 +496,8 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation if not isinstance(last_name_array, (list, tuple)): last_name_array = [] + # if not positive_value_exists(invitation_message): + # invitation_message = "" if email_address_array: # Reconstruct dictionary array from lists for n in range(len(email_address_array)): @@ -2023,6 +2025,25 @@ def generate_mutual_friends_for_all_voters(): if positive_value_exists(mutual_friends_update_suppressed_count): total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + queryset = FriendInvitationVoterLink.objects.all() + queryset = queryset.filter(deleted=False) + queryset = queryset.order_by('-date_last_changed') + friend_invitation_voter_link_list = list(queryset) + # Loop through them SuggestedFriend list + for one_friend_invitation_voter_link in friend_invitation_voter_link_list: + generate_results = generate_mutual_friends_for_friend_invitation_voter_link( + friend_invitation_voter_link=one_friend_invitation_voter_link) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + if positive_value_exists(mutual_friends_created_count): + total_mutual_friends_created_count += mutual_friends_created_count + if positive_value_exists(mutual_friends_updated_count): + total_mutual_friends_updated_count += mutual_friends_updated_count + if positive_value_exists(mutual_friends_update_suppressed_count): + total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + if positive_value_exists(total_mutual_friends_created_count): status += "created: " + str(total_mutual_friends_created_count) + " " if positive_value_exists(total_mutual_friends_updated_count): @@ -2041,7 +2062,8 @@ def generate_mutual_friends_for_one_voter(voter_we_vote_id='', update_existing_d status = "" success = True friend_manager = FriendManager() - voter_manager = VoterManager() + + # ###################### # Retrieve list of all CurrentFriend entries connected to voter_we_vote_id current_friend_list_results = friend_manager.retrieve_current_friend_list( voter_we_vote_id=voter_we_vote_id, @@ -2050,21 +2072,55 @@ def generate_mutual_friends_for_one_voter(voter_we_vote_id='', update_existing_d status += current_friend_list_results['status'] success = False results = { - 'success': success, - 'status': status, + 'success': success, + 'status': status, } return results - current_friend_list = current_friend_list_results['current_friend_list'] - - # Loop through them CurrentFriend list for one_current_friend in current_friend_list: generate_results = generate_mutual_friends_for_current_friend(current_friend=one_current_friend) status += generate_results['status'] + # ###################### + # Retrieve list of all SuggestedFriend entries connected to voter_we_vote_id + suggested_friend_list_results = friend_manager.retrieve_suggested_friend_list( + voter_we_vote_id=voter_we_vote_id, + read_only=False) + if not suggested_friend_list_results['success']: + status += suggested_friend_list_results['status'] + success = False + results = { + 'success': success, + 'status': status, + } + return results + suggested_friend_list = suggested_friend_list_results['suggested_friend_list'] + for one_suggested_friend in suggested_friend_list: + generate_results = generate_mutual_friends_for_suggested_friend(suggested_friend=one_suggested_friend) + status += generate_results['status'] + + # ###################### + # Retrieve list of all FriendInvitationVoterLink entries connected to voter_we_vote_id + sent_to_me_results = friend_manager.retrieve_friend_invitations_sent_to_me( + recipient_voter_we_vote_id=voter_we_vote_id, + read_only=False) + if not sent_to_me_results['success']: + status += sent_to_me_results['status'] + success = False + results = { + 'success': success, + 'status': status, + } + return results + friend_invitation_voter_link_list = sent_to_me_results['friend_list'] + for one_friend_invitation_voter_link in friend_invitation_voter_link_list: + generate_results = generate_mutual_friends_for_friend_invitation_voter_link( + friend_invitation_voter_link=one_friend_invitation_voter_link) + status += generate_results['status'] + results = { - 'success': success, - 'status': status, + 'success': success, + 'status': status, } return results @@ -2145,6 +2201,87 @@ def generate_mutual_friends_for_current_friend(current_friend=None, update_exist return results +def generate_mutual_friends_for_friend_invitation_voter_link( + friend_invitation_voter_link=None, + update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id + friend_voter_we_vote_id = friend_invitation_voter_link.recipient_voter_we_vote_id + + # Retrieve the voter_we_vote_id's of all mutual friends, based on query of CurrentFriend table + mutual_voter_we_vote_id_list_from_current_friends = \ + friend_manager.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id=voter_we_vote_id, + friend_voter_we_vote_id=friend_voter_we_vote_id) + + # Update SuggestedFriend with mutual_friend_count + mutual_friend_count = len(mutual_voter_we_vote_id_list_from_current_friends) + # if positive_value_exists(mutual_friend_count): + # status += "MUTUAL_FRIEND_COUNT: " + str(mutual_friend_count) + " " + + generate_results = generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + mutual_friends_voter_we_vote_id_list_from_current_friends=mutual_voter_we_vote_id_list_from_current_friends, + second_friend_voter_we_vote_id=friend_voter_we_vote_id, + update_existing_data=update_existing_data) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + + change_to_save = False + mutual_friend_count_change = False + if not positive_value_exists(mutual_friend_count) and friend_invitation_voter_link.mutual_friend_count is None: + pass + elif friend_invitation_voter_link.mutual_friend_count != mutual_friend_count: + friend_invitation_voter_link.mutual_friend_count = mutual_friend_count + friend_invitation_voter_link.mutual_friend_count_last_updated = localtime(now()).date() # Pacific Time + + change_to_save = True + mutual_friend_count_change = True + + always_update_serialized = True + if always_update_serialized or mutual_friend_count_change or update_existing_data: + # Note that in the SuggestedFriend table, we store both "directions" of the suggested friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B + preview_results = generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + second_friend_voter_we_vote_id=friend_voter_we_vote_id) + status += preview_results['status'] + if preview_results['success']: + mutual_friend_preview_list_serialized = preview_results['mutual_friend_preview_list_serialized'] + if not positive_value_exists(mutual_friend_preview_list_serialized) and \ + friend_invitation_voter_link.mutual_friend_preview_list_serialized is None: + pass + elif friend_invitation_voter_link.mutual_friend_preview_list_serialized != \ + mutual_friend_preview_list_serialized: + friend_invitation_voter_link.mutual_friend_preview_list_serialized = \ + mutual_friend_preview_list_serialized + friend_invitation_voter_link.mutual_friend_preview_list_update_needed = False + change_to_save = True + else: + status += "FAILED_TO_GENERATE_PREVIEW_LIST_FRIEND_INVITATION_VOTER_LINK " + + if change_to_save: + friend_invitation_voter_link.save() + + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + def generate_mutual_friends_for_suggested_friend(suggested_friend=None, update_existing_data=False): status = "" success = True @@ -2533,6 +2670,7 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): status += retrieve_invitations_sent_to_me_results['status'] if retrieve_invitations_sent_to_me_results['friend_list_found']: raw_friend_list = retrieve_invitations_sent_to_me_results['friend_list'] + friend_invitation_dict = {} heal_results = heal_friend_invitations_sent_to_me(voter.we_vote_id, raw_friend_list) invitation_table_dict = {} verified_friend_list = heal_results['friend_list'] @@ -2540,6 +2678,10 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): sent_to_me_voter_we_vote_id_list = [] recipient_voter_email_dict = {} for one_friend_invitation in verified_friend_list: + friend_invitation_dict[one_friend_invitation.sender_voter_we_vote_id] = \ + one_friend_invitation \ + if hasattr(one_friend_invitation, "mutual_friend_count") \ + else "" sent_to_me_voter_we_vote_id_list.append(one_friend_invitation.sender_voter_we_vote_id) invitation_table_dict[one_friend_invitation.sender_voter_we_vote_id] = \ one_friend_invitation.invitation_table \ @@ -2558,6 +2700,26 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): for friend_voter in sent_to_me_friend_list: # This is the voter who sent the invitation to me mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( voter.we_vote_id, friend_voter.we_vote_id) + if friend_voter.we_vote_id in friend_invitation_dict: + mutual_friend_count = friend_invitation_dict[friend_voter.we_vote_id].mutual_friend_count \ + if hasattr(friend_invitation_dict[friend_voter.we_vote_id], "mutual_friend_count") and \ + positive_value_exists(friend_invitation_dict[friend_voter.we_vote_id].mutual_friend_count) \ + else 0 + mutual_friend_preview_list_serialized = \ + friend_invitation_dict[friend_voter.we_vote_id].mutual_friend_preview_list_serialized \ + if hasattr(friend_invitation_dict[friend_voter.we_vote_id], + "mutual_friend_preview_list_serialized") \ + and positive_value_exists(friend_invitation_dict[friend_voter.we_vote_id] + .mutual_friend_preview_list_serialized) \ + else None + if mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + else: + mutual_friend_count = 0 + mutual_friend_preview_list = [] # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) one_friend = { @@ -2578,6 +2740,8 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): "invitation_status": "", # Not used for invitations sent to me "invitation_sent_to": recipient_voter_email_dict[friend_voter.we_vote_id], "invitation_table": invitation_table_dict[friend_voter.we_vote_id], + "mutual_friend_count": mutual_friend_count, + "mutual_friend_preview_list": mutual_friend_preview_list, "mutual_friends": mutual_friends, # "positions_taken": positions_taken, } @@ -2630,10 +2794,12 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): status += retrieve_invitations_sent_by_me_results['status'] if retrieve_invitations_sent_by_me_results['friend_list_found']: raw_friend_list = retrieve_invitations_sent_by_me_results['friend_list'] + friend_invitation_dict = {} invitation_status_dict = {} invitation_table_dict = {} recipient_voter_email_dict = {} recipient_voter_email_invitations = [] + recipient_voter_we_vote_id = '' recipient_voter_we_vote_id_list = [] for one_friend_invitation in raw_friend_list: # Two kinds of invitations come in the raw_friend_list, 1) an invitation connected to voter @@ -2643,6 +2809,10 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): else: recipient_voter_we_vote_id = "" if positive_value_exists(recipient_voter_we_vote_id): + friend_invitation_dict[recipient_voter_we_vote_id] = \ + one_friend_invitation \ + if hasattr(one_friend_invitation, "mutual_friend_count") \ + else "" recipient_voter_we_vote_id_list.append(recipient_voter_we_vote_id) invitation_status_dict[recipient_voter_we_vote_id] = \ one_friend_invitation.invitation_status \ @@ -2668,7 +2838,28 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( - voter.we_vote_id, friend_voter.we_vote_id) + voter.we_vote_id, recipient_voter_we_vote_id) + if recipient_voter_we_vote_id in friend_invitation_dict: + mutual_friend_count = friend_invitation_dict[recipient_voter_we_vote_id].mutual_friend_count \ + if hasattr(friend_invitation_dict[recipient_voter_we_vote_id], "mutual_friend_count") and \ + positive_value_exists(friend_invitation_dict[recipient_voter_we_vote_id] + .mutual_friend_count) \ + else 0 + mutual_friend_preview_list_serialized = \ + friend_invitation_dict[recipient_voter_we_vote_id].mutual_friend_preview_list_serialized \ + if hasattr(friend_invitation_dict[recipient_voter_we_vote_id], + "mutual_friend_preview_list_serialized") \ + and positive_value_exists(friend_invitation_dict[recipient_voter_we_vote_id] + .mutual_friend_preview_list_serialized) \ + else None + if mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + else: + mutual_friend_count = 0 + mutual_friend_preview_list = [] one_friend = { "voter_we_vote_id": friend_voter.we_vote_id, "voter_date_last_changed": friend_voter.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), @@ -2687,6 +2878,8 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): "invitation_status": invitation_status_dict[friend_voter.we_vote_id], "invitation_table": invitation_table_dict[friend_voter.we_vote_id], "invitation_sent_to": recipient_voter_email_dict[friend_voter.we_vote_id], + "mutual_friend_count": mutual_friend_count, + "mutual_friend_preview_list": mutual_friend_preview_list, "mutual_friends": mutual_friends, # "positions_taken": positions_taken, } diff --git a/friend/models.py b/friend/models.py index 27fb0fe9e..a23167948 100644 --- a/friend/models.py +++ b/friend/models.py @@ -168,6 +168,13 @@ class FriendInvitationVoterLink(models.Model): date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True, db_index=True) merge_by_secret_key_allowed = models.BooleanField(default=True) # To allow merges after delete invitation_table = models.CharField(max_length=8, default=FRIEND_INVITATION_VOTER_LINK) + + mutual_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + mutual_friend_count_last_updated = models.DateTimeField(null=True) + + mutual_friend_preview_list_serialized = models.TextField(default=None, null=True) + mutual_friend_preview_list_update_needed = models.BooleanField(default=True) + deleted = models.BooleanField(default=False) # If invitation is completed or rescinded, mark as deleted diff --git a/voter/controllers_contacts.py b/voter/controllers_contacts.py index 141962271..2d5b1581d 100644 --- a/voter/controllers_contacts.py +++ b/voter/controllers_contacts.py @@ -56,9 +56,20 @@ def move_voter_contact_email_to_another_voter(from_voter_we_vote_id, to_voter_we # ###################### # Migrations try: + query = VoterContactEmail.objects.all() + query = query.filter(imported_by_voter_we_vote_id__iexact=to_voter_we_vote_id) + query = query.exclude(google_contact_id__isnull=True) + query = query.values_list('google_contact_id', flat=True).distinct() + google_contact_id_list_to_not_overwrite = list(query) + voter_contact_email_entries_moved += VoterContactEmail.objects\ .filter(imported_by_voter_we_vote_id__iexact=from_voter_we_vote_id)\ + .exclude(google_contact_id__in=google_contact_id_list_to_not_overwrite)\ .update(imported_by_voter_we_vote_id=to_voter_we_vote_id) + + entries_deleted = \ + VoterContactEmail.objects.filter(imported_by_voter_we_vote_id__iexact=from_voter_we_vote_id).delete() + status += "ENTRIES_DELETED: " + str(entries_deleted) + " " except Exception as e: status += "FAILED-VOTER_CONTACT_EMAIL_UPDATE_IMPORTED_BY: " + str(e) + " " @@ -217,6 +228,7 @@ def voter_contact_list_retrieve_for_api(voter_we_vote_id=''): # voterContactLis if hasattr(voter_contact_email, 'google_contact_id') else '', 'google_date_last_updated': google_date_last_updated_string, 'has_data_from_google_people_api': voter_contact_email.has_data_from_google_people_api, + 'id': voter_contact_email.id if hasattr(voter_contact_email, 'id') else 0, 'ignore_contact': voter_contact_email.ignore_contact, 'imported_by_voter_we_vote_id': voter_contact_email.imported_by_voter_we_vote_id if hasattr(voter_contact_email, 'imported_by_voter_we_vote_id') else '', From 51873c0806bb1327e511196318bf5123b0cfddf5 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Thu, 23 Jun 2022 18:34:58 -0700 Subject: [PATCH 19/98] Broke up the steps of voterContactListSave so we can provide feedback as the process proceeds. --- apis_v1/views/views_voter.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index 194517a92..534478565 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -3056,11 +3056,18 @@ def voter_contact_list_save_view(request): # voterContactListSave status, voter, voter_found, voter_device_link = views_voter_utils.get_voter_from_request(request, status) contacts_string = request.POST.get('contacts', None) - augment_voter_contact_emails_with_location = request.POST.get('augment_voter_contact_emails_with_location', False) + + augment_with_location = request.POST.get('augment_voter_contact_emails_with_location', False) + augment_with_location = positive_value_exists(augment_with_location) + + augment_with_we_vote_data = request.POST.get('augment_voter_contact_emails_with_we_vote_data', False) + augment_with_we_vote_data = positive_value_exists(augment_with_we_vote_data) + delete_all_voter_contact_emails = request.POST.get('delete_all_voter_contact_emails', False) + delete_all_voter_contact_emails = positive_value_exists(delete_all_voter_contact_emails) google_api_key_type = request.POST.get('google_api_key_type', 'ballot') - if positive_value_exists(delete_all_voter_contact_emails): + if delete_all_voter_contact_emails: results = delete_all_voter_contact_emails_for_voter(voter_we_vote_id=voter.we_vote_id) elif hasattr(voter, 'we_vote_id') and contacts_string: contacts = json.loads(contacts_string) @@ -3068,9 +3075,10 @@ def voter_contact_list_save_view(request): # voterContactListSave results = save_google_contacts(voter_we_vote_id=voter.we_vote_id, contacts=contacts) status += results['status'] - from email_outbound.controllers import augment_emails_for_voter_with_we_vote_data - augment_results = augment_emails_for_voter_with_we_vote_data(voter_we_vote_id=voter.we_vote_id) - status += augment_results['status'] + if augment_with_we_vote_data: + from email_outbound.controllers import augment_emails_for_voter_with_we_vote_data + augment_results = augment_emails_for_voter_with_we_vote_data(voter_we_vote_id=voter.we_vote_id) + status += augment_results['status'] # 2021-09-30 Requires Pro account which costs $90/month # 2022-06-06 We are currently paying for this SendGrid account, so we can implement this @@ -3083,7 +3091,7 @@ def voter_contact_list_save_view(request): # voterContactListSave # augment_results = augment_emails_for_voter_with_snovio(voter_we_vote_id=voter.we_vote_id) # status += augment_results['status'] - if positive_value_exists(augment_voter_contact_emails_with_location): + if augment_with_location: from import_export_open_people.controllers import augment_emails_for_voter_with_open_people augment_results = augment_emails_for_voter_with_open_people(voter_we_vote_id=voter.we_vote_id) status += augment_results['status'] @@ -3092,12 +3100,17 @@ def voter_contact_list_save_view(request): # voterContactListSave voter_contact_email_list = retrieve_results['voter_contact_email_list'] voter_contact_email_google_count = retrieve_results['voter_contact_email_google_count'] + augment_sequence_complete = delete_all_voter_contact_emails or augment_with_location + augment_sequence_has_next_step = augment_with_we_vote_data or (contacts_stored > 0) json_data = { 'status': status, 'success': success, - 'augment_voter_contact_emails_with_location': augment_voter_contact_emails_with_location, + 'augment_voter_contact_emails_with_location': augment_with_location, + 'augment_voter_contact_emails_with_we_vote_data': augment_with_we_vote_data, 'contacts_stored': contacts_stored, 'delete_all_voter_contact_emails': delete_all_voter_contact_emails, + 'voter_contact_email_augment_sequence_complete': augment_sequence_complete, + 'voter_contact_email_augment_sequence_has_next_step': augment_sequence_has_next_step, 'voter_contact_email_google_count': voter_contact_email_google_count, 'voter_contact_email_list': voter_contact_email_list, 'voter_contact_email_list_count': len(voter_contact_email_list), From 3871c5c96ec5959c37e2a20bbb9844d9558d7f0e Mon Sep 17 00:00:00 2001 From: snyk-bot Date: Tue, 5 Jul 2022 03:37:54 +0000 Subject: [PATCH 20/98] fix: requirements.txt to reduce vulnerabilities The following vulnerabilities are fixed by pinning transitive dependencies: - https://snyk.io/vuln/SNYK-PYTHON-DJANGO-2940618 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f1321dae2..d214f9cb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ boto3==1.13.16 cryptography==3.4.8 dj-database-url==0.5.0 dj-static==0.0.6 -django==3.2.13 +django==3.2.14 django-background-tasks==1.2.5 django-bootstrap3==15.0.0 django-cors-headers==3.7.0 From c138fa0abffbbce683aa2a22a3930ef4e35c071d Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Thu, 7 Jul 2022 07:35:29 -0700 Subject: [PATCH 21/98] Sped up some "move" functions to make the friend acceptance process faster. Moved the "your friend accepted your friend invitation" into a follow-up API call, so it doesn't delay. --- apis_v1/views/views_friend.py | 44 ++- ballot/controllers.py | 125 ++++---- follow/controllers.py | 36 +-- follow/models.py | 130 +++++--- friend/controllers.py | 562 +++++++++++++++++++++++----------- friend/models.py | 18 +- position/models.py | 416 ++++++++++++------------- voter/controllers.py | 20 +- voter_guide/controllers.py | 15 +- wevote_functions/functions.py | 2 + 10 files changed, 809 insertions(+), 559 deletions(-) diff --git a/apis_v1/views/views_friend.py b/apis_v1/views/views_friend.py index 6aa62519d..8349a1075 100644 --- a/apis_v1/views/views_friend.py +++ b/apis_v1/views/views_friend.py @@ -3,7 +3,8 @@ # -*- coding: UTF-8 -*- from config.base import get_environment_variable from django.http import HttpResponse -from friend.controllers import friend_invitation_by_email_send_for_api, friend_invitation_by_email_verify_for_api, \ +from friend.controllers import friend_acceptance_email_should_be_sent, \ + friend_invitation_by_email_send_for_api, friend_invitation_by_email_verify_for_api, \ friend_invitation_by_we_vote_id_send_for_api, friend_invite_response_for_api, friend_list_for_api, \ friend_lists_all_for_api, friend_invitation_by_facebook_send_for_api, \ friend_invitation_by_facebook_verify_for_api, friend_invitation_information_for_api, message_to_friend_send_for_api @@ -57,19 +58,38 @@ def friend_invitation_by_email_verify_view(request): # friendInvitationByEmailV :return: """ voter_device_id = get_voter_device_id(request) # We standardize how we take in the voter_device_id + acceptance_email_should_be_sent = positive_value_exists(request.GET.get('acceptance_email_should_be_sent', False)) invitation_secret_key = request.GET.get('invitation_secret_key', "") hostname = request.GET.get('hostname', "") - results = friend_invitation_by_email_verify_for_api(voter_device_id, invitation_secret_key, - web_app_root_url=hostname) - json_data = { - 'status': results['status'], - 'success': results['success'], - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': results['voter_has_data_to_preserve'], - 'invitation_found': results['invitation_found'], - 'attempted_to_approve_own_invitation': results['attempted_to_approve_own_invitation'], - 'invitation_secret_key': invitation_secret_key, - } + if acceptance_email_should_be_sent: + results = friend_acceptance_email_should_be_sent( + voter_device_id, + invitation_secret_key, + web_app_root_url=hostname) + json_data = { + 'status': results['status'], + 'success': results['success'], + 'acceptance_email_should_be_sent': results['acceptance_email_should_be_sent'], + 'attempted_to_approve_own_invitation': results['attempted_to_approve_own_invitation'], + 'invitation_found': results['invitation_found'], + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': results['voter_has_data_to_preserve'], + } + else: + results = friend_invitation_by_email_verify_for_api( + voter_device_id, + invitation_secret_key) + json_data = { + 'status': results['status'], + 'success': results['success'], + 'acceptance_email_should_be_sent': results['acceptance_email_should_be_sent'], + 'attempted_to_approve_own_invitation': results['attempted_to_approve_own_invitation'], + 'invitation_found': results['invitation_found'], + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': results['voter_has_data_to_preserve'], + } return HttpResponse(json.dumps(json_data), content_type='application/json') diff --git a/ballot/controllers.py b/ballot/controllers.py index 2db874b45..041bf2c58 100644 --- a/ballot/controllers.py +++ b/ballot/controllers.py @@ -1011,28 +1011,28 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve if not positive_value_exists(voter_id): status += " " + "VALID_VOTER_ID_MISSING" error_json_data = { - 'status': status, - 'success': False, - 'ballot_caveat': '', - 'ballot_found': False, - 'ballot_item_list': [], + 'status': status, + 'success': False, + 'ballot_caveat': '', + 'ballot_found': False, + 'ballot_item_list': [], 'ballot_location_display_name': '', 'ballot_location_shortcut': ballot_location_shortcut, 'ballot_returned_we_vote_id': ballot_returned_we_vote_id, - 'google_civic_election_id': google_civic_election_id, - 'is_from_substituted_address': False, - 'is_from_test_ballot': False, + 'google_civic_election_id': google_civic_election_id, + 'is_from_substituted_address': False, + 'is_from_test_ballot': False, 'next_national_election_day_text': '', - 'original_text_city': '', - 'original_text_state': '', - 'original_text_zip': '', + 'original_text_city': '', + 'original_text_state': '', + 'original_text_zip': '', 'polling_location_we_vote_id_source': '', - 'substituted_address_nearby': '', - 'substituted_address_city': '', - 'substituted_address_state': '', - 'substituted_address_zip': '', - 'text_for_map_search': '', - 'voter_device_id': voter_device_id, + 'substituted_address_nearby': '', + 'substituted_address_city': '', + 'substituted_address_state': '', + 'substituted_address_zip': '', + 'text_for_map_search': '', + 'voter_device_id': voter_device_id, } return error_json_data @@ -1043,6 +1043,9 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve status += " " + voter_address_results['status'] # Note that this might be an empty VoterAddress object voter_address = voter_address_results['voter_address'] + text_for_map_search = '' + if voter_address and hasattr(voter_address, 'text_for_map_search'): + text_for_map_search = voter_address.text_for_map_search if positive_value_exists(voter_address_results['voter_address_has_value']): ballot_retrieval_based_on_voter_address = True else: @@ -1062,35 +1065,35 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve elif positive_value_exists(ballot_location_shortcut): ballot_caveat = "We could not find the ballot '{ballot_location_shortcut}'.".format( ballot_location_shortcut=ballot_location_shortcut) - elif positive_value_exists(voter_address.text_for_map_search): + elif positive_value_exists(text_for_map_search): ballot_caveat = "We could not find a ballot near '{text_for_map_search}'.".format( - text_for_map_search=voter_address.text_for_map_search) + text_for_map_search=text_for_map_search) else: ballot_caveat = "Please save your address so we can find your ballot." error_json_data = { - 'status': status, - 'success': True, - 'ballot_caveat': ballot_caveat, - 'ballot_found': False, - 'ballot_item_list': [], + 'status': status, + 'success': True, + 'ballot_caveat': ballot_caveat, + 'ballot_found': False, + 'ballot_item_list': [], 'ballot_location_display_name': '', 'ballot_location_shortcut': ballot_location_shortcut, 'ballot_returned_we_vote_id': ballot_returned_we_vote_id, - 'google_civic_election_id': google_civic_election_id, - 'is_from_substituted_address': False, - 'is_from_test_ballot': False, + 'google_civic_election_id': google_civic_election_id, + 'is_from_substituted_address': False, + 'is_from_test_ballot': False, 'next_national_election_day_text': next_national_election_day_text, - 'original_text_city': '', - 'original_text_state': '', - 'original_text_zip': '', - 'polling_location_we_vote_id_source': '', - 'substituted_address_nearby': '', - 'substituted_address_city': '', - 'substituted_address_state': '', - 'substituted_address_zip': '', - 'text_for_map_search': voter_address.text_for_map_search, - 'voter_device_id': voter_device_id, + 'original_text_city': '', + 'original_text_state': '', + 'original_text_zip': '', + 'polling_location_we_vote_id_source': '', + 'substituted_address_nearby': '', + 'substituted_address_city': '', + 'substituted_address_state': '', + 'substituted_address_zip': '', + 'text_for_map_search': text_for_map_search, + 'voter_device_id': voter_device_id, } return error_json_data @@ -1166,34 +1169,34 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve status += " " + results['status'] json_data = { - 'status': status, - 'success': True, - 'ballot_caveat': voter_ballot_saved.ballot_caveat(), - 'ballot_found': True, - 'ballot_item_list': results['ballot_item_list'], - 'ballot_location_display_name': voter_ballot_saved.ballot_location_display_name, - 'ballot_location_shortcut': voter_ballot_saved.ballot_location_shortcut, - 'ballot_returned_we_vote_id': voter_ballot_saved.ballot_returned_we_vote_id, - 'election_name': voter_ballot_saved.election_description_text, - 'election_day_text': voter_ballot_saved.election_day_text(), - 'google_civic_election_id': google_civic_election_id, - 'is_from_substituted_address': voter_ballot_saved.is_from_substituted_address, - 'is_from_test_ballot': voter_ballot_saved.is_from_test_ballot, + 'status': status, + 'success': True, + 'ballot_caveat': voter_ballot_saved.ballot_caveat(), + 'ballot_found': True, + 'ballot_item_list': results['ballot_item_list'], + 'ballot_location_display_name': voter_ballot_saved.ballot_location_display_name, + 'ballot_location_shortcut': voter_ballot_saved.ballot_location_shortcut, + 'ballot_returned_we_vote_id': voter_ballot_saved.ballot_returned_we_vote_id, + 'election_name': voter_ballot_saved.election_description_text, + 'election_day_text': voter_ballot_saved.election_day_text(), + 'google_civic_election_id': google_civic_election_id, + 'is_from_substituted_address': voter_ballot_saved.is_from_substituted_address, + 'is_from_test_ballot': voter_ballot_saved.is_from_test_ballot, 'next_national_election_day_text': next_national_election_day_text, - 'original_text_city': voter_ballot_saved.original_text_city, - 'original_text_state': voter_ballot_saved.original_text_state, - 'original_text_zip': voter_ballot_saved.original_text_zip, - 'polling_location_we_vote_id_source': voter_ballot_saved.polling_location_we_vote_id_source, - 'substituted_address_nearby': voter_ballot_saved.substituted_address_nearby, - 'substituted_address_city': voter_ballot_saved.substituted_address_city, - 'substituted_address_state': voter_ballot_saved.substituted_address_state, - 'substituted_address_zip': voter_ballot_saved.substituted_address_zip, - 'text_for_map_search': voter_ballot_saved.original_text_for_map_search, - 'voter_device_id': voter_device_id, + 'original_text_city': voter_ballot_saved.original_text_city, + 'original_text_state': voter_ballot_saved.original_text_state, + 'original_text_zip': voter_ballot_saved.original_text_zip, + 'polling_location_we_vote_id_source': voter_ballot_saved.polling_location_we_vote_id_source, + 'substituted_address_nearby': voter_ballot_saved.substituted_address_nearby, + 'substituted_address_city': voter_ballot_saved.substituted_address_city, + 'substituted_address_state': voter_ballot_saved.substituted_address_state, + 'substituted_address_zip': voter_ballot_saved.substituted_address_zip, + 'text_for_map_search': voter_ballot_saved.original_text_for_map_search, + 'voter_device_id': voter_device_id, } return json_data - status += " " + "NO_VOTER_BALLOT_SAVED_FOUND" + status += " " + "NO_VOTER_BALLOT_SAVED_FOUND " json_data = { 'status': status, 'success': True, diff --git a/follow/controllers.py b/follow/controllers.py index e3b1ef0c6..1343728f0 100644 --- a/follow/controllers.py +++ b/follow/controllers.py @@ -227,7 +227,7 @@ def duplicate_follow_issue_entries_to_another_voter(from_voter_we_vote_id, to_vo return results -def move_follow_entries_to_another_voter(from_voter_id, to_voter_id, to_voter_we_vote_id): +def move_follow_entries_to_another_voter(from_voter_id=0, to_voter_id=0, to_voter_we_vote_id=''): status = '' success = False follow_entries_moved = 0 @@ -260,23 +260,23 @@ def move_follow_entries_to_another_voter(from_voter_id, to_voter_id, to_voter_we return results follow_organization_list = FollowOrganizationList() - follow_organization_manager = FollowOrganizationManager() - from_follow_list = follow_organization_list.retrieve_follow_organization_by_voter_id(from_voter_id) - - for from_follow_entry in from_follow_list: - # See if the "to_voter" already has an entry for this organization - existing_entry_results = follow_organization_manager.retrieve_follow_organization( - 0, to_voter_id, from_follow_entry.organization_id, from_follow_entry.organization_we_vote_id) - if not existing_entry_results['follow_organization_found']: - # Change the voter_id and voter_we_vote_id - try: - from_follow_entry.voter_id = to_voter_id - # We don't currently store follow entries by we_vote_id - # from_follow_entry.voter_we_vote_id = to_voter_we_vote_id - from_follow_entry.save() - follow_entries_moved += 1 - except Exception as e: - follow_entries_not_moved += 1 + organization_we_vote_ids_followed = \ + follow_organization_list.retrieve_follow_organization_by_voter_id_simple_id_array( + voter_id=to_voter_id, + return_we_vote_id=True, + ) + + move_results = follow_organization_list.move_follow_organization_from_voter_id_to_new_voter_id( + from_voter_id=from_voter_id, + to_voter_id=to_voter_id, + exclude_organization_we_vote_id_list=organization_we_vote_ids_followed, + ) + follow_entries_moved = move_results['number_moved'] + + if move_results['success']: + # Finally, delete remaining FollowOrganization entries for from_voter_id + delete_results = follow_organization_list.delete_follow_organization_list_for_voter_id(voter_id=from_voter_id) + follow_entries_not_moved = delete_results['number_deleted'] results = { 'status': status, diff --git a/follow/models.py b/follow/models.py index 37e3d3633..2e681aac4 100644 --- a/follow/models.py +++ b/follow/models.py @@ -1248,7 +1248,6 @@ def retrieve_follow_organization_by_voter_id(self, voter_id, auto_followed_from_ read_only=False): # Retrieve a list of follow_organization entries for this voter follow_organization_list_found = False - following_status = FOLLOWING follow_organization_list = {} try: # Should not default to 'readonly' since we sometimes save the results of this call @@ -1257,7 +1256,7 @@ def retrieve_follow_organization_by_voter_id(self, voter_id, auto_followed_from_ else: follow_organization_list = FollowOrganization.objects.all() follow_organization_list = follow_organization_list.filter(voter_id=voter_id) - follow_organization_list = follow_organization_list.filter(following_status=following_status) + follow_organization_list = follow_organization_list.filter(following_status=FOLLOWING) if auto_followed_from_twitter_suggestion: follow_organization_list = follow_organization_list.filter( auto_followed_from_twitter_suggestion=auto_followed_from_twitter_suggestion) @@ -1272,6 +1271,58 @@ def retrieve_follow_organization_by_voter_id(self, voter_id, auto_followed_from_ follow_organization_list = {} return follow_organization_list + def delete_follow_organization_list_for_voter_id( + self, + voter_id=0, + ): + number_deleted = 0 + success = True + status = '' + try: + queryset = FollowOrganization.objects.all() + queryset = queryset.filter(voter_id=voter_id) + number_deleted = queryset.delete() + status += "DELETED_FOLLOW_ORGANIZATION: " + str(number_deleted) + " " + except Exception as e: + success = False + status += "FAILED_TO_DELETE_FOLLOW_ORGANIZATION: " + str(e) + " " + + results = { + 'success': success, + 'status': status, + 'number_deleted': number_deleted, + } + return results + + def move_follow_organization_from_voter_id_to_new_voter_id( + self, + from_voter_id=0, + to_voter_id=0, + exclude_organization_we_vote_id_list=[], + ): + success = True + status = '' + try: + queryset = FollowOrganization.objects.all() + queryset = queryset.filter(voter_id=from_voter_id) + if len(exclude_organization_we_vote_id_list): + queryset = queryset.exclude(organization_we_vote_id__in=exclude_organization_we_vote_id_list) + number_moved = queryset.update( + voter_id=to_voter_id, + ) + status += "UPDATED_FOLLOW_ORGANIZATION: " + str(number_moved) + " " + except Exception as e: + number_moved = 0 + success = False + status += "FAILED_TO_UPDATE_FOLLOW_ORGANIZATION: " + str(e) + " " + + results = { + 'success': success, + 'status': status, + 'number_moved': number_moved, + } + return results + def retrieve_follow_organization_by_own_organization_we_vote_id(self, organization_we_vote_id, auto_followed_from_twitter_suggestion=False): # Retrieve a list of followed organizations entries by voter_linked_organization_we_vote_id for voter guides @@ -1320,36 +1371,27 @@ def retrieve_ignore_organization_by_voter_id(self, voter_id, read_only=False): follow_organization_list = {} return follow_organization_list - def retrieve_follow_organization_by_voter_id_simple_id_array(self, voter_id, return_we_vote_id=False, - auto_followed_from_twitter_suggestion=False, - read_only=False): - follow_organization_list_manager = FollowOrganizationList() - follow_organization_list = \ - follow_organization_list_manager.retrieve_follow_organization_by_voter_id( - voter_id, auto_followed_from_twitter_suggestion, read_only=read_only) - follow_organization_list_simple_array = [] - if len(follow_organization_list): - voter_manager = VoterManager() - voter_linked_organization_we_vote_id = \ - voter_manager.fetch_linked_organization_we_vote_id_from_local_id(voter_id) - for follow_organization in follow_organization_list: - if not read_only: - # Heal the data by making sure the voter's linked_organization_we_vote_id exists and is accurate - if positive_value_exists(voter_linked_organization_we_vote_id) \ - and voter_linked_organization_we_vote_id != \ - follow_organization.voter_linked_organization_we_vote_id: - try: - follow_organization.voter_linked_organization_we_vote_id = \ - voter_linked_organization_we_vote_id - follow_organization.save() - except Exception as e: - status = 'FAILED_TO_UPDATE_FOLLOW_ISSUE-voter_id ' + str(voter_id) - handle_record_not_saved_exception(e, logger=logger, exception_message_optional=status) + def retrieve_follow_organization_by_voter_id_simple_id_array( + self, + voter_id=0, + return_we_vote_id=False, + auto_followed_from_twitter_suggestion=False): + + try: + queryset = FollowOrganization.objects.using('readonly').all() + queryset = queryset.filter(voter_id=voter_id) + queryset = queryset.filter(following_status=FOLLOWING) + if auto_followed_from_twitter_suggestion: + queryset = queryset.filter( + auto_followed_from_twitter_suggestion=auto_followed_from_twitter_suggestion) + if positive_value_exists(return_we_vote_id): + queryset = queryset.values_list('organization_we_vote_id', flat=True).distinct() + else: + queryset = queryset.values_list('organization_id', flat=True).distinct() + follow_organization_list_simple_array = list(queryset) + except Exception as e: + follow_organization_list_simple_array = [] - if return_we_vote_id: - follow_organization_list_simple_array.append(follow_organization.organization_we_vote_id) - else: - follow_organization_list_simple_array.append(follow_organization.organization_id) return follow_organization_list_simple_array def retrieve_followed_organization_by_organization_we_vote_id_simple_id_array( @@ -1419,18 +1461,20 @@ def retrieve_followers_organization_by_organization_we_vote_id_simple_id_array( return followers_organization_list_simple_array def retrieve_ignore_organization_by_voter_id_simple_id_array( - self, voter_id, return_we_vote_id=False, read_only=False): - follow_organization_list_manager = FollowOrganizationList() - ignore_organization_list = \ - follow_organization_list_manager.retrieve_ignore_organization_by_voter_id(voter_id, read_only=read_only) - ignore_organization_list_simple_array = [] - if len(ignore_organization_list): - for ignore_organization in ignore_organization_list: - if return_we_vote_id: - ignore_organization_list_simple_array.append(ignore_organization.organization_we_vote_id) - else: - ignore_organization_list_simple_array.append(ignore_organization.organization_id) - return ignore_organization_list_simple_array + self, voter_id, return_we_vote_id=False): + try: + queryset = FollowOrganization.objects.using('readonly').all() + queryset = queryset.filter(voter_id=voter_id) + queryset = queryset.filter(following_status=FOLLOW_IGNORE) + if positive_value_exists(return_we_vote_id): + queryset = queryset.values_list('organization_we_vote_id', flat=True).distinct() + else: + queryset = queryset.values_list('organization_id', flat=True).distinct() + follow_organization_list_simple_array = list(queryset) + except Exception as e: + follow_organization_list_simple_array = [] + + return follow_organization_list_simple_array def retrieve_follow_organization_by_organization_id(self, organization_id): # Retrieve a list of follow_organization entries for this organization diff --git a/friend/controllers.py b/friend/controllers.py index 0edade095..c675494f9 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -11,7 +11,7 @@ from config.base import get_environment_variable from email_outbound.controllers import schedule_email_with_email_outbound_description, schedule_verification_email from email_outbound.models import EmailAddress, EmailManager, FRIEND_ACCEPTED_INVITATION_TEMPLATE, \ - FRIEND_INVITATION_TEMPLATE, MESSAGE_TO_FRIEND_TEMPLATE, TO_BE_PROCESSED, WAITING_FOR_VERIFICATION + FRIEND_INVITATION_TEMPLATE, MESSAGE_TO_FRIEND_TEMPLATE, SENT, TO_BE_PROCESSED, WAITING_FOR_VERIFICATION from follow.models import FollowIssueList from import_export_facebook.models import FacebookManager import json @@ -204,8 +204,11 @@ def fetch_friend_invitation_recipient_voter_we_vote_id(friend_invitation): return '' -def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_we_vote_id, invitation_message='', - web_app_root_url=''): +def friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + invitation_message='', + web_app_root_url=''): """ A person has accepted a friend request, so we want to email the original_sender voter who invited the accepting_voter @@ -331,6 +334,15 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ send_results = email_manager.send_scheduled_email(email_scheduled) email_scheduled_sent = send_results['email_scheduled_sent'] status += send_results['status'] + if email_scheduled_sent: + # If scheduled email sent successfully change their status from WAITING_FOR_VERIFICATION to SENT + send_status = SENT + try: + email_scheduled.send_status = send_status + email_scheduled.save() + except Exception as e: + status += "ERROR_FAILED_TO_UPDATE_FRIEND_ACCEPTED_SEND_STATUS: " + str(e) + ' ' + print(status) results = { 'success': True, @@ -799,7 +811,7 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key friend_we_vote_id = '' friend_organization_we_vote_id = '' invitation_message = '' - invitation_secret_key_belongs_to_this_voter = '' + invitation_secret_key_belongs_to_this_voter = True # If a voter_device_id is passed in that isn't valid, we want to throw an error device_id_results = is_voter_device_id_valid(voter_device_id) @@ -807,20 +819,20 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key status += "friendInvitationInformation-MISSING_VOTER_DEVICE_ID:" status += device_id_results['status'] json_data = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return json_data @@ -830,17 +842,17 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key 'status': status, 'success': False, 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results @@ -853,17 +865,17 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key 'status': status, 'success': False, 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results voter = voter_results['voter'] @@ -871,24 +883,26 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key friend_manager = FriendManager() friend_invitation_results = friend_manager.retrieve_friend_invitation_from_secret_key( - invitation_secret_key, for_retrieving_information=True, read_only=True) + invitation_secret_key, + for_retrieving_information=True, + read_only=True) if not friend_invitation_results['friend_invitation_found']: status += "INVITATION_NOT_FOUND_FROM_SECRET_KEY-RETRIEVING_INFO " error_results = { - 'status': status, - 'success': True, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': True, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results @@ -896,72 +910,72 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key invitation_found = True if friend_invitation_results['friend_invitation_voter_link_found']: friend_invitation_voter_link = friend_invitation_results['friend_invitation_voter_link'] - status += "FRIEND_INVITATION_VOTER_LINK_FOUND " + status += "INVITATION_INFORMATION_FRIEND_INVITATION_VOTER_LINK_FOUND " + sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id - if friend_invitation_voter_link.sender_voter_we_vote_id == voter_we_vote_id: + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED-VOTER_LINK " error_results = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': invitation_found, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': invitation_found, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results if friend_invitation_voter_link.recipient_voter_we_vote_id != voter_we_vote_id: status += "RECIPIENT_DOES_NOT_MATCH_CURRENT_VOTER-VOTER_LINK " error_results = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': invitation_found, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': invitation_found, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results - invitation_secret_key_belongs_to_this_voter = True - sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id invitation_message = friend_invitation_voter_link.invitation_message elif friend_invitation_results['friend_invitation_email_link_found']: friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] - status += "FRIEND_INVITATION_EMAIL_LINK_FOUND " + status += "INVITATION_INFORMATION_FRIEND_INVITATION_EMAIL_LINK_FOUND " + sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id - if friend_invitation_email_link.sender_voter_we_vote_id == voter_we_vote_id: + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED-EMAIL_LINK " error_results = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': invitation_found, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': invitation_found, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results @@ -987,35 +1001,37 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key status += "MISSING_SENDER_VOTER_WE_VOTE_ID " json_data = { - 'status': status, - 'success': success, - 'voter_device_id': voter_device_id, - 'friend_first_name': friend_first_name, - 'friend_last_name': friend_last_name, - 'friend_image_url_https_large': friend_image_url_https_large, - 'friend_image_url_https_tiny': friend_image_url_https_tiny, - 'friend_issue_we_vote_id_list': friend_issue_we_vote_id_list, - 'friend_we_vote_id': friend_we_vote_id, - 'friend_organization_we_vote_id': friend_organization_we_vote_id, - 'invitation_found': invitation_found, - 'invitation_message': invitation_message, - 'invitation_secret_key': invitation_secret_key, + 'status': status, + 'success': success, + 'voter_device_id': voter_device_id, + 'friend_first_name': friend_first_name, + 'friend_last_name': friend_last_name, + 'friend_image_url_https_large': friend_image_url_https_large, + 'friend_image_url_https_tiny': friend_image_url_https_tiny, + 'friend_issue_we_vote_id_list': friend_issue_we_vote_id_list, + 'friend_we_vote_id': friend_we_vote_id, + 'friend_organization_we_vote_id': friend_organization_we_vote_id, + 'invitation_found': invitation_found, + 'invitation_message': invitation_message, + 'invitation_secret_key': invitation_secret_key, 'invitation_secret_key_belongs_to_this_voter': invitation_secret_key_belongs_to_this_voter, } return json_data -def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify +def friend_acceptance_email_should_be_sent( # friendInvitationByEmailVerify voter_device_id, invitation_secret_key, web_app_root_url=''): """ - + Friendship has been completed and new voter is signed in. Now we want to start the + process of letting the person who invited this friend know that their friend accepted + the invitation. :param voter_device_id: :param invitation_secret_key: :param web_app_root_url: :return: """ status = "" - success = False + success = True # If a voter_device_id is passed in that isn't valid, we want to throw an error device_id_results = is_voter_device_id_valid(voter_device_id) @@ -1024,11 +1040,12 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify json_data = { 'status': status, 'success': False, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': False, - 'invitation_found': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, } return json_data @@ -1037,11 +1054,12 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify error_results = { 'status': status, 'success': True, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': False, - 'invitation_found': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, } return error_results @@ -1053,11 +1071,191 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify error_results = { 'status': status, 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, 'voter_device_id': voter_device_id, 'voter_has_data_to_preserve': False, + } + return error_results + voter = voter_results['voter'] + voter_we_vote_id = voter.we_vote_id + voter_has_data_to_preserve = voter.has_data_to_preserve() + + friend_manager = FriendManager() + friend_invitation_results = friend_manager.retrieve_friend_invitation_from_secret_key( + invitation_secret_key, + for_additional_processes=True, + read_only=False) + if not friend_invitation_results['friend_invitation_found']: + status += "INVITATION_NOT_FOUND_FROM_SECRET_KEY3-SEND_ACCEPTANCE_EMAIL " + error_results = { + 'status': status, + 'success': True, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + # Now that we know we are dealing with a valid friend invitation, extract the info we need + invitation_found = False + voter_we_vote_id_accepting_invitation = "" + if friend_invitation_results['friend_invitation_voter_link_found']: + invitation_found = True + status += "FRIEND_ACCEPTANCE_FRIEND_INVITATION_VOTER_LINK_FOUND " + friend_invitation_voter_link = friend_invitation_results['friend_invitation_voter_link'] + sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id + voter_we_vote_id_accepting_invitation = friend_invitation_voter_link.recipient_voter_we_vote_id + + if sender_voter_we_vote_id == voter_we_vote_id: + status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " + error_results = { + 'status': status, + 'success': True, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': True, + 'invitation_found': invitation_found, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) + friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + + if not positive_value_exists(friend_invitation_voter_link.invited_friend_accepted_notification_sent): + accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation + original_sender_we_vote_id = sender_voter_we_vote_id + results = friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + web_app_root_url=web_app_root_url) + status += results['status'] + try: + friend_invitation_voter_link.invited_friend_accepted_notification_sent = True + friend_invitation_voter_link.save() + except Exception as e: + status += "COULD_NOT_SAVE_FRIEND_INVITATION_VOTER_LINK: " + str(e) + " " + else: + status += "ALREADY_TRUE: friend_invitation_voter_link.invited_friend_accepted_notification_sent " + elif friend_invitation_results['friend_invitation_email_link_found']: + invitation_found = True + status += "FRIEND_ACCEPTANCE_FRIEND_INVITATION_EMAIL_LINK_FOUND " + friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] + sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id + voter_we_vote_id_accepting_invitation = friend_invitation_email_link.recipient_voter_we_vote_id + + if sender_voter_we_vote_id == voter_we_vote_id: + status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " + error_results = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': True, + 'invitation_found': invitation_found, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) + friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + + if not positive_value_exists(friend_invitation_email_link.invited_friend_accepted_notification_sent): + accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation + original_sender_we_vote_id = sender_voter_we_vote_id + results = friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + web_app_root_url=web_app_root_url) + status = results['status'] + try: + friend_invitation_email_link.invited_friend_accepted_notification_sent = True + friend_invitation_email_link.save() + except Exception as e: + status += "COULD_NOT_SAVE_FRIEND_INVITATION_EMAIL_LINK: " + str(e) + " " + else: + status += "ALREADY_TRUE: friend_invitation_email_link.invited_friend_accepted_notification_sent " + else: + invitation_found = False + status += "FRIEND_INVITATION_NOT_FOUND " + + json_data = { + 'status': status, + 'success': success, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': invitation_found, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return json_data + + +def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify + voter_device_id, invitation_secret_key): + """ + + :param voter_device_id: + :param invitation_secret_key: + :return: + """ + status = "" + success = True + + # If a voter_device_id is passed in that isn't valid, we want to throw an error + device_id_results = is_voter_device_id_valid(voter_device_id) + if not device_id_results['success']: + status += "DEVICE_ID_RESULTS_SUCCESS_FALSE " + status += device_id_results['status'] + json_data = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, + } + return json_data + + if not positive_value_exists(invitation_secret_key): + status += "VOTER_EMAIL_ADDRESS_VERIFY_MISSING_SECRET_KEY " + error_results = { + 'status': status, + 'success': True, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, + } + return error_results + + voter_manager = VoterManager() + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_id = voter_results['voter_id'] + if not positive_value_exists(voter_id): + status += "VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID " + error_results = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, } return error_results voter = voter_results['voter'] @@ -1066,41 +1264,47 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify friend_manager = FriendManager() friend_invitation_results = friend_manager.retrieve_friend_invitation_from_secret_key( - invitation_secret_key, for_accepting_friendship=True, read_only=False) + invitation_secret_key, + for_accepting_friendship=True, + read_only=False) if not friend_invitation_results['friend_invitation_found']: status += "INVITATION_NOT_FOUND_FROM_SECRET_KEY2-ACCEPTING_FRIENDSHIP " error_results = { 'status': status, 'success': True, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return error_results # Now that we have the friend_invitation data, look more closely at it - invitation_found = True - voter_we_vote_id_accepting_invitation = "" + acceptance_email_should_be_sent = False + invitation_found = False email_manager = EmailManager() if friend_invitation_results['friend_invitation_voter_link_found']: friend_invitation_voter_link = friend_invitation_results['friend_invitation_voter_link'] + sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id + voter_we_vote_id_accepting_invitation = friend_invitation_voter_link.recipient_voter_we_vote_id + invitation_found = True - if friend_invitation_voter_link.sender_voter_we_vote_id == voter_we_vote_id: + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " error_results = { 'status': status, 'success': True, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': True, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': True, + 'invitation_found': True, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return error_results - voter_we_vote_id_accepting_invitation = friend_invitation_voter_link.recipient_voter_we_vote_id # Now we want to make sure we have a current_friend entry recipient_organization_we_vote_id = '' voter_results = voter_manager.retrieve_voter_by_we_vote_id( @@ -1108,21 +1312,12 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify if voter_results['voter_found']: recipient_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( - sender_voter_we_vote_id=friend_invitation_voter_link.sender_voter_we_vote_id, - recipient_voter_we_vote_id=friend_invitation_voter_link.recipient_voter_we_vote_id, + sender_voter_we_vote_id=sender_voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, sender_organization_we_vote_id=voter.linked_organization_we_vote_id, recipient_organization_we_vote_id=recipient_organization_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter( - friend_invitation_voter_link.sender_voter_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter( - friend_invitation_voter_link.recipient_voter_we_vote_id) - - accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation - original_sender_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id - results = friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_we_vote_id, - web_app_root_url=web_app_root_url) - status += results['status'] + acceptance_email_should_be_sent = True # Now that a CurrentFriend entry exists, update the FriendInvitation... if friend_results['success']: @@ -1131,7 +1326,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify friend_invitation_voter_link.save() except Exception as e: success = False - status += 'FAILED_TO_UPDATE_INVITATION_STATUS1 ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_INVITATION_STATUS1: ' + str(e) + ' ' else: success = False status += "friend_invitation_voter_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " @@ -1139,16 +1334,20 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # We don't need to do anything with the email because this was an invitation to a known voter elif friend_invitation_results['friend_invitation_email_link_found']: friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] - if friend_invitation_email_link.sender_voter_we_vote_id == voter_we_vote_id: + sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id + invitation_found = True + + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " error_results = { 'status': status, 'success': False, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': True, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': True, + 'invitation_found': True, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return error_results @@ -1205,7 +1404,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify update_voter_name = True except Exception as e: success = False - status += 'FAILED_TO_UPDATE_UNVERIFIED_EMAIL ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_UNVERIFIED_EMAIL: ' + str(e) + ' ' else: email_ownership_is_verified = True email_create_results = email_manager.create_email_address_for_voter( @@ -1233,26 +1432,16 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # Now that we know who owns the recipient_email_address, update invitation status sender_organization_we_vote_id = '' - voter_results = voter_manager.retrieve_voter_by_we_vote_id( - friend_invitation_email_link.sender_voter_we_vote_id) + voter_results = voter_manager.retrieve_voter_by_we_vote_id(sender_voter_we_vote_id) if voter_results['voter_found']: sender_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( - friend_invitation_email_link.sender_voter_we_vote_id, - voter_we_vote_id_accepting_invitation, - sender_organization_we_vote_id, - voter.linked_organization_we_vote_id - ) - - friend_manager.update_suggested_friends_starting_with_one_voter( - friend_invitation_email_link.sender_voter_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + sender_voter_we_vote_id=sender_voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, + sender_organization_we_vote_id=sender_organization_we_vote_id, + recipient_organization_we_vote_id=voter.linked_organization_we_vote_id) - accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation - original_sender_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id - results = friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_we_vote_id, - web_app_root_url=web_app_root_url) - status = results['status'] + acceptance_email_should_be_sent = True if friend_results['success']: try: @@ -1262,7 +1451,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify status += ' friend_invitation_email_link_found FRIENDSHIP_CREATED ' except Exception as e: success = False - status += 'FAILED_TO_UPDATE_INVITATION_STATUS2 ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_INVITATION_STATUS2: ' + str(e) + ' ' else: success = False status += "friend_invitation_email_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " @@ -1288,16 +1477,20 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify voter.save() status += "VOTER_AND_ORGANIZATION_CREATED_FROM_FRIEND_INVITATION " except Exception as e: - status += "UNABLE_CREATE_AND_LINK_VOTER_FROM_FRIEND_INVITATION " + str(e) + ' ' + status += "UNABLE_CREATE_AND_LINK_VOTER_FROM_FRIEND_INVITATION: " + str(e) + ' ' + else: + acceptance_email_should_be_sent = False + status += "BOTH_FALSE: friend_invitation_email_link_found and friend_invitation_voter_link_found " json_data = { 'status': status, 'success': success, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': invitation_found, + 'acceptance_email_should_be_sent': acceptance_email_should_be_sent, 'attempted_to_approve_own_invitation': False, + 'invitation_found': invitation_found, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return json_data @@ -1503,11 +1696,10 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque if voter_results['voter_found']: sender_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( - sender_voter_we_vote_id, - voter_we_vote_id_accepting_invitation, - sender_organization_we_vote_id, - voter.linked_organization_we_vote_id, - ) + sender_voter_we_vote_id=sender_voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, + sender_organization_we_vote_id=sender_organization_we_vote_id, + recipient_organization_we_vote_id=voter.linked_organization_we_vote_id) friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) @@ -1522,7 +1714,7 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque status += "INVITATION_FROM_FACEBOOK_UPDATED " except Exception as e: success = False - status += 'FAILED_TO_UPDATE_INVITATION_STATUS1 ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_INVITATION_STATUS1: ' + str(e) + ' ' else: success = False status += " friend_invitation_facebook_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " @@ -3303,7 +3495,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ from_sender_entry.save() friend_invitation_entries_moved += 1 except Exception as e: - status += "FriendInvitationEmailLink Sender entries not moved " + str(e) + ' ' + status += "FriendInvitationEmailLink Sender entries not moved: " + str(e) + ' ' friend_invitation_entries_not_moved += 1 else: status += "to_sender_invitation_found found, EmailLink Sender entries not moved " @@ -3342,7 +3534,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ from_sender_entry.save() friend_invitation_entries_moved += 1 except Exception as e: - status += "FriendInvitationVoterLink Sender entries not moved " + str(e) + ' ' + status += "FriendInvitationVoterLink Sender entries not moved: " + str(e) + ' ' friend_invitation_entries_not_moved += 1 else: status += "to_sender_invitation_found found, VoterLink Sender entries not moved " @@ -3467,7 +3659,7 @@ def move_friends_to_another_voter( friend_entries_moved += 1 except Exception as e: friend_entries_not_moved += 1 - status += "PROBLEM_UPDATING_FRIEND " + str(e) + ' ' + status += "PROBLEM_UPDATING_FRIEND: " + str(e) + ' ' from_friend_list_remaining_results = friend_manager.retrieve_current_friend_list( from_voter_we_vote_id, @@ -3478,7 +3670,7 @@ def move_friends_to_another_voter( try: from_friend_entry.delete() except Exception as e: - status += "PROBLEM_DELETING_FRIEND " + str(e) + ' ' + status += "PROBLEM_DELETING_FRIEND: " + str(e) + ' ' results = { 'status': status, @@ -3552,7 +3744,7 @@ def move_suggested_friends_to_another_voter( suggested_friend_entries_moved += 1 except Exception as e: suggested_friend_entries_not_moved += 1 - status += "PROBLEM_UPDATING_SUGGESTED_FRIEND " + str(e) + ' ' + status += "PROBLEM_UPDATING_SUGGESTED_FRIEND: " + str(e) + ' ' from_friend_list_remaining_results = friend_manager.retrieve_suggested_friend_list( from_voter_we_vote_id, hide_deleted=False, read_only=False) @@ -3563,7 +3755,7 @@ def move_suggested_friends_to_another_voter( # Leave this turned off until testing is finished from_friend_entry.delete() except Exception as e: - status += "PROBLEM_DELETING_FRIEND " + str(e) + ' ' + status += "PROBLEM_DELETING_FRIEND: " + str(e) + ' ' results = { 'status': status, diff --git a/friend/models.py b/friend/models.py index a23167948..28a1312eb 100644 --- a/friend/models.py +++ b/friend/models.py @@ -106,6 +106,7 @@ class FriendInvitationEmailLink(models.Model): recipient_last_name = models.CharField(verbose_name='last name', max_length=255, null=True, blank=True) secret_key = models.CharField( verbose_name="secret key to accept invite", max_length=255, null=True, blank=True, unique=True) + invited_friend_accepted_notification_sent = models.BooleanField(default=False) invitation_message = models.TextField(null=True, blank=True) invitation_status = models.CharField(max_length=50, choices=INVITATION_STATUS_CHOICES, default=NO_RESPONSE) date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) @@ -163,6 +164,7 @@ class FriendInvitationVoterLink(models.Model): max_length=255, null=True, blank=True, unique=False, db_index=True) secret_key = models.CharField( verbose_name="secret key to accept invite", max_length=255, null=True, blank=True, unique=True) + invited_friend_accepted_notification_sent = models.BooleanField(default=False) invitation_message = models.TextField(null=True, blank=True) invitation_status = models.CharField(max_length=50, choices=INVITATION_STATUS_CHOICES, default=NO_RESPONSE) date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True, db_index=True) @@ -1897,12 +1899,18 @@ def retrieve_friend_invitations_sent_to_me(self, recipient_voter_we_vote_id, rea return results def retrieve_friend_invitation_from_secret_key( - self, invitation_secret_key, for_accepting_friendship=False, for_merge_accounts=False, - for_retrieving_information=False, read_only=True): + self, + invitation_secret_key, + for_accepting_friendship=False, + for_additional_processes=False, + for_merge_accounts=False, + for_retrieving_information=False, + read_only=True): """ :param invitation_secret_key: :param for_accepting_friendship: + :param for_additional_processes: :param for_merge_accounts: :param for_retrieving_information: :param read_only: @@ -1927,6 +1935,9 @@ def retrieve_friend_invitation_from_secret_key( if positive_value_exists(for_accepting_friendship): voter_link_query = voter_link_query.exclude(invitation_status__iexact=ACCEPTED) status += "FOR_ACCEPTING_FRIENDSHIP " + elif positive_value_exists(for_additional_processes): + voter_link_query = voter_link_query.filter(invitation_status__iexact=ACCEPTED) + status += "FOR_ADDITIONAL_PROCESSES " elif positive_value_exists(for_merge_accounts): voter_link_query = voter_link_query.filter(merge_by_secret_key_allowed=True) status += "FOR_MERGE_ACCOUNTS " @@ -1974,6 +1985,9 @@ def retrieve_friend_invitation_from_secret_key( if positive_value_exists(for_accepting_friendship): email_link_query = email_link_query.exclude(invitation_status__iexact=ACCEPTED) status += "FOR_ACCEPTING_FRIENDSHIP " + elif positive_value_exists(for_additional_processes): + email_link_query = email_link_query.filter(invitation_status__iexact=ACCEPTED) + status += "FOR_ADDITIONAL_PROCESSES " elif positive_value_exists(for_merge_accounts): email_link_query = email_link_query.filter(merge_by_secret_key_allowed=True) status += "FOR_MERGE_ACCOUNTS " diff --git a/position/models.py b/position/models.py index 8166325a5..b66bebdba 100644 --- a/position/models.py +++ b/position/models.py @@ -1462,242 +1462,218 @@ def repair_all_positions_for_voter(self, incoming_voter_id=0, incoming_voter_we_ failure_counter = 0 ############################ - # Retrieve public positions + # Repair positions owned by voter_id + public_number_changed = 0 + friend_number_changed = 0 try: - # Retrieve by voter_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter(voter_id=voter_id) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() - - except Exception as e: - failure_counter += 1 - - # Retrieve by voter_we_vote_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter( - voter_we_vote_id__iexact=voter_we_vote_id) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() - - except Exception as e: - failure_counter += 1 - - # Retrieve by organization_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter(organization_id=organization_id) - # As of Aug 2018 we are no longer using PERCENT_RATING - public_positions_list_query = public_positions_list_query.exclude(stance__iexact=PERCENT_RATING) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() - - except Exception as e: - failure_counter += 1 - - # Retrieve by organization_we_vote_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter( - organization_we_vote_id__iexact=organization_we_vote_id) - # As of Aug 2018 we are no longer using PERCENT_RATING - public_positions_list_query = public_positions_list_query.exclude(stance__iexact=PERCENT_RATING) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() + # ...without the right voter_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_id=voter_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_id=voter_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right organization_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + # ...without the right organization_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) except Exception as e: - results = { - 'status': 'REPAIR-VOTER_POSITION_FOR_PUBLIC_SEARCH_FAILED ', - 'success': False, - 'repair_complete': False, - } - return results + failure_counter += 1 ############################ - # Retrieve positions meant for friends only + # Repair public positions owned by voter_we_vote_id try: - # Retrieve by voter_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter(voter_id=voter_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + # ...without the right voter_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right organization_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) - # Retrieve by voter_we_vote_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter( - voter_we_vote_id__iexact=voter_we_vote_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + # ...without the right organization_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + except Exception as e: + failure_counter += 1 - except Exception as e: - failure_counter += 1 + ############################ + # Repair public positions owned by organization_id + try: + # ...without the right voter_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) - # Retrieve by organization_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter(organization_id=organization_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + # ...without the right voter_we_vot_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right organization_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_id=organization_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_id=organization_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + except Exception as e: + failure_counter += 1 - # Retrieve by organization_we_vote_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter( - organization_we_vote_id__iexact=organization_we_vote_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + ############################ + # Repair public positions owned by organization_we_vote_id + try: + # ...without the right voter_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right voter_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + # ...without the right organization_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) except Exception as e: - results = { - 'status': 'REPAIR-VOTER_POSITION_FOR_FRIENDS_SEARCH_FAILED ', - 'success': False, - 'repair_complete': False, - } - return results + failure_counter += 1 if positive_value_exists(failure_counter): results = { diff --git a/voter/controllers.py b/voter/controllers.py index 1f768c4e6..1085b064d 100644 --- a/voter/controllers.py +++ b/voter/controllers.py @@ -162,7 +162,7 @@ def delete_all_voter_information_permanently(voter_to_delete=None): # voterDele from_linked_organization = from_linked_organization_results['organization'] voter_to_delete_linked_organization_id = from_linked_organization.id else: - # Remove the link to the organization so we don't have a future conflict + # Remove the link to the organization, so we don't have a future conflict try: voter_to_delete_linked_organization_we_vote_id = None voter_to_delete.linked_organization_we_vote_id = None @@ -176,7 +176,7 @@ def delete_all_voter_information_permanently(voter_to_delete=None): # voterDele delete_apple_user_results = delete_apple_user_entries_for_voter(voter_to_delete_we_vote_id) status += delete_apple_user_results['status'] - # Data healing scripts before we try to move the positions + # Data healing scripts before we try to delete the positions position_list_manager = PositionListManager() if positive_value_exists(voter_to_delete_id): repair_results = position_list_manager.repair_all_positions_for_voter(voter_to_delete_id) @@ -204,7 +204,7 @@ def delete_all_voter_information_permanently(voter_to_delete=None): # voterDele status += " " + delete_friend_invitations_results['status'] if positive_value_exists(voter_to_delete.linked_organization_we_vote_id): - # Remove the link to the organization so we don't have a future conflict + # Remove the link to the organization, so we don't have a future conflict try: voter_to_delete.linked_organization_we_vote_id = None voter_to_delete.save() @@ -981,7 +981,7 @@ def move_facebook_info_to_another_voter(from_voter, to_voter): temp_facebook_id = from_voter.facebook_id temp_facebook_profile_image_url_https = from_voter.facebook_profile_image_url_https temp_fb_username = from_voter.fb_username - # Now delete it and save so we can save the unique facebook_id in the to_voter + # Now delete it and save, so we can save the unique facebook_id in the to_voter from_voter.facebook_email = "" from_voter.facebook_id = 0 from_voter.facebook_profile_image_url_https = "" @@ -1094,7 +1094,7 @@ def move_twitter_info_to_another_voter(from_voter, to_voter): temp_twitter_name = from_voter.twitter_name temp_twitter_profile_image_url_https = from_voter.twitter_profile_image_url_https temp_twitter_screen_name = from_voter.twitter_screen_name - # Now delete it and save so we can save the unique facebook_id in the to_voter + # Now delete it and save, so we can save the unique facebook_id in the to_voter from_voter.twitter_id = None from_voter.twitter_name = "" from_voter.twitter_profile_image_url_https = "" @@ -2311,7 +2311,7 @@ def voter_merge_two_accounts_action( # voterMergeTwoAccounts, part 2 from_linked_organization = from_linked_organization_results['organization'] from_voter_linked_organization_id = from_linked_organization.id else: - # Remove the link to the organization so we don't have a future conflict + # Remove the link to the organization, so we don't have a future conflict try: from_voter_linked_organization_we_vote_id = None from_voter.linked_organization_we_vote_id = None @@ -2329,7 +2329,7 @@ def voter_merge_two_accounts_action( # voterMergeTwoAccounts, part 2 to_linked_organization = to_linked_organization_results['organization'] to_voter_linked_organization_id = to_linked_organization.id else: - # Remove the link to the organization so we don't have a future conflict + # Remove the link to the organization, so we don't have a future conflict try: to_voter_linked_organization_we_vote_id = None new_owner_voter.linked_organization_we_vote_id = None @@ -2401,7 +2401,7 @@ def voter_merge_two_accounts_action( # voterMergeTwoAccounts, part 2 status += " " + move_friend_invitations_results['status'] if positive_value_exists(from_voter.linked_organization_we_vote_id): - # Remove the link to the organization so we don't have a future conflict + # Remove the link to the organization, so we don't have a future conflict try: from_voter.linked_organization_we_vote_id = None from_voter.save() @@ -3363,7 +3363,7 @@ def refresh_voter_primary_email_cached_information_by_email(normalized_email_add status += "UNABLE_TO_UPDATE_VOTER_FOUND_BY_EMAIL " # We tried to update the voter found by the voter_we_vote_id stored in the EmailAddress table, # but got an error, so assume it was because of a collision with the primary_email_we_vote_id - # Here, we retrieve the voter already "claiming" this email entry so we can wipe the + # Here, we retrieve the voter already "claiming" this email entry, so we can wipe the # email values. voter_by_primary_email_results = voter_manager.retrieve_voter_by_primary_email_we_vote_id( verified_email_address_object.we_vote_id) @@ -3444,7 +3444,7 @@ def refresh_voter_primary_email_cached_information_by_email(normalized_email_add # email address removed. if positive_value_exists(email_results['success']): # Make sure no voter's think they are using this email address - # Remove the email information so we don't have a future conflict + # Remove the email information, so we don't have a future conflict try: if voter_found_by_email_boolean: voter_found_by_email.email = None diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index e88a600de..7a8247673 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -3109,8 +3109,7 @@ def retrieve_voter_guides_to_follow_by_ballot_item(voter_id, kind_of_ballot_item follow_organization_list_manager = FollowOrganizationList() organizations_followed_by_voter = \ - follow_organization_list_manager.retrieve_follow_organization_by_voter_id_simple_id_array(voter_id, - read_only=True) + follow_organization_list_manager.retrieve_follow_organization_by_voter_id_simple_id_array(voter_id) positions_list = position_list_manager.calculate_positions_not_followed_by_voter( all_positions_list, organizations_followed_by_voter) @@ -3236,10 +3235,10 @@ def retrieve_voter_guides_to_follow_by_election_for_api(voter_id, google_civic_e return_we_vote_id = True organization_we_vote_ids_followed_by_voter = \ follow_organization_list_manager.retrieve_follow_organization_by_voter_id_simple_id_array( - voter_id, return_we_vote_id, read_only=True) + voter_id, return_we_vote_id) organization_we_vote_ids_ignored_by_voter = \ follow_organization_list_manager.retrieve_ignore_organization_by_voter_id_simple_id_array( - voter_id, return_we_vote_id, read_only=True) + voter_id, return_we_vote_id) # position_list_manager = PositionListManager() if not positive_value_exists(google_civic_election_id): @@ -3602,10 +3601,10 @@ def retrieve_voter_guides_to_follow_generic_for_api(voter_id, search_string, fil read_only = True organization_we_vote_ids_followed_by_voter = \ follow_organization_list_manager.retrieve_follow_organization_by_voter_id_simple_id_array( - voter_id, return_we_vote_id, read_only=read_only) + voter_id, return_we_vote_id) organization_we_vote_ids_ignored_by_voter = \ follow_organization_list_manager.retrieve_ignore_organization_by_voter_id_simple_id_array( - voter_id, return_we_vote_id, read_only=read_only) + voter_id, return_we_vote_id) # This is a list of orgs that the voter is already following or ignoring organization_we_vote_ids_followed_or_ignored_by_voter = list(chain(organization_we_vote_ids_followed_by_voter, @@ -5077,8 +5076,8 @@ def retrieve_voter_guides_ignored(voter_id): # voterGuidesIgnoredRetrieve follow_organization_list_manager = FollowOrganizationList() return_we_vote_id = True organization_we_vote_ids_ignored_by_voter = \ - follow_organization_list_manager.retrieve_ignore_organization_by_voter_id_simple_id_array(voter_id, - return_we_vote_id) + follow_organization_list_manager.retrieve_ignore_organization_by_voter_id_simple_id_array( + voter_id, return_we_vote_id) voter_guide_list_object = VoterGuideListManager() results = voter_guide_list_object.retrieve_voter_guides_by_organization_list( diff --git a/wevote_functions/functions.py b/wevote_functions/functions.py index 616ee903d..dd508236a 100644 --- a/wevote_functions/functions.py +++ b/wevote_functions/functions.py @@ -1522,6 +1522,8 @@ def positive_value_exists(value): return False except ValueError: return False + except Exception as e: + return False return bool(value) From c5db306d3415df1700834567f928aee9480a5b8c Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Thu, 7 Jul 2022 12:33:39 -0700 Subject: [PATCH 22/98] Fixed crashing bug. --- friend/controllers.py | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/friend/controllers.py b/friend/controllers.py index c675494f9..aa31ca93b 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -1103,8 +1103,6 @@ def friend_acceptance_email_should_be_sent( # friendInvitationByEmailVerify return error_results # Now that we know we are dealing with a valid friend invitation, extract the info we need - invitation_found = False - voter_we_vote_id_accepting_invitation = "" if friend_invitation_results['friend_invitation_voter_link_found']: invitation_found = True status += "FRIEND_ACCEPTANCE_FRIEND_INVITATION_VOTER_LINK_FOUND " @@ -1149,7 +1147,7 @@ def friend_acceptance_email_should_be_sent( # friendInvitationByEmailVerify status += "FRIEND_ACCEPTANCE_FRIEND_INVITATION_EMAIL_LINK_FOUND " friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id - voter_we_vote_id_accepting_invitation = friend_invitation_email_link.recipient_voter_we_vote_id + voter_we_vote_id_accepting_invitation = voter_we_vote_id if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " @@ -1166,23 +1164,26 @@ def friend_acceptance_email_should_be_sent( # friendInvitationByEmailVerify return error_results friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) - - if not positive_value_exists(friend_invitation_email_link.invited_friend_accepted_notification_sent): - accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation - original_sender_we_vote_id = sender_voter_we_vote_id - results = friend_accepted_invitation_send( - accepting_voter_we_vote_id, - original_sender_we_vote_id, - web_app_root_url=web_app_root_url) - status = results['status'] - try: - friend_invitation_email_link.invited_friend_accepted_notification_sent = True - friend_invitation_email_link.save() - except Exception as e: - status += "COULD_NOT_SAVE_FRIEND_INVITATION_EMAIL_LINK: " + str(e) + " " + if positive_value_exists(voter_we_vote_id_accepting_invitation): + friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + + if not positive_value_exists(friend_invitation_email_link.invited_friend_accepted_notification_sent): + accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation + original_sender_we_vote_id = sender_voter_we_vote_id + results = friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + web_app_root_url=web_app_root_url) + status = results['status'] + try: + friend_invitation_email_link.invited_friend_accepted_notification_sent = True + friend_invitation_email_link.save() + except Exception as e: + status += "COULD_NOT_SAVE_FRIEND_INVITATION_EMAIL_LINK: " + str(e) + " " + else: + status += "ALREADY_TRUE: friend_invitation_email_link.invited_friend_accepted_notification_sent " else: - status += "ALREADY_TRUE: friend_invitation_email_link.invited_friend_accepted_notification_sent " + status += "FRIEND_INVITATION_EMAIL_LINK_MISSING-voter_we_vote_id_accepting_invitation " else: invitation_found = False status += "FRIEND_INVITATION_NOT_FOUND " From 441f16752374f36e17fee5c5d483f2d0f08701a7 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Fri, 8 Jul 2022 13:12:21 -0700 Subject: [PATCH 23/98] Testing fix where email not marked as signed in. --- friend/controllers.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/friend/controllers.py b/friend/controllers.py index aa31ca93b..880c61748 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -1308,8 +1308,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # Now we want to make sure we have a current_friend entry recipient_organization_we_vote_id = '' - voter_results = voter_manager.retrieve_voter_by_we_vote_id( - friend_invitation_voter_link.recipient_voter_we_vote_id) + voter_results = voter_manager.retrieve_voter_by_we_vote_id(voter_we_vote_id_accepting_invitation) if voter_results['voter_found']: recipient_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( @@ -1371,6 +1370,9 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # We might need to heal the data in the voter record if voter_we_vote_id_accepting_invitation != voter_we_vote_id: + status += "VOTER_ACCEPTING_INVITATION_NOT_CURRENT_SIGNED_IN_VOTER " + email_address_object.email_ownership_is_verified = True + email_address_object.save() email_owner_results = voter_manager.retrieve_voter_by_we_vote_id(email_address_object.voter_we_vote_id) if email_owner_results['voter_found']: email_owner_voter = email_owner_results['voter'] @@ -1378,6 +1380,9 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify else: # If we are here, then the email_address_object doesn't belong to another voter and can be # claimed by this current voter. + status += "VOTER_ACCEPTING_INVITATION_IS_CURRENT_SIGNED_IN_VOTER " + email_address_object.email_ownership_is_verified = True + email_address_object.save() voter_manager.update_voter_email_ownership_verified(voter, email_address_object) if we_have_first_or_last_name_from_friend_invitation_email_link and \ not this_voter_has_first_or_last_name_saved: @@ -1385,6 +1390,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify update_voter_name = True else: + status += "VERIFIED_EMAIL_ADDRESS_OBJECT_NOT_FOUND " voter_we_vote_id_accepting_invitation = voter_we_vote_id # If we are here, we know the email is unclaimed. We can assign it to the current voter. # Is there an email address entry for this voter/email? @@ -1393,6 +1399,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify friend_invitation_email_link.recipient_voter_email, email_we_vote_id, voter_we_vote_id) if email_results['email_address_object_found']: + status += "UNVERIFIED_EMAIL_ADDRESS_OBJECT_FOUND " email_address_object = email_results['email_address_object'] try: email_address_object.email_ownership_is_verified = True @@ -1407,6 +1414,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify success = False status += 'FAILED_TO_UPDATE_UNVERIFIED_EMAIL: ' + str(e) + ' ' else: + status += "UNVERIFIED_EMAIL_ADDRESS_OBJECT_NOT_FOUND " email_ownership_is_verified = True email_create_results = email_manager.create_email_address_for_voter( friend_invitation_email_link.recipient_voter_email, voter, email_ownership_is_verified) From d5e48103f90594f1a309077f61c9bbfba0f83f90 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Sat, 9 Jul 2022 16:52:58 -0700 Subject: [PATCH 24/98] Supporting delete_all_voter_information_permanently triggered from WebApp. Updates to how we connect to SendGrid. Added retrieve date to elections list. --- apis_v1/views/views_voter.py | 26 +++++++- election/views_admin.py | 5 ++ email_outbound/models.py | 59 +++++++++++-------- requirements.txt | 3 +- templates/election/election_list.html | 6 ++ .../email_templates/friend_invitation.html | 5 +- .../email_templates/sign_in_code_email.html | 1 - .../email_templates/sign_in_code_email.txt | 4 +- voter/controllers.py | 2 +- 9 files changed, 78 insertions(+), 33 deletions(-) diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index 534478565..3456a9522 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -33,7 +33,8 @@ from sms.models import SMSManager from support_oppose_deciding.controllers import voter_opposing_save, voter_stop_opposing_save, \ voter_stop_supporting_save, voter_supporting_save_for_api -from voter.controllers import voter_address_retrieve_for_api, voter_create_for_api, voter_merge_two_accounts_for_api, \ +from voter.controllers import delete_all_voter_information_permanently, \ + voter_address_retrieve_for_api, voter_create_for_api, voter_merge_two_accounts_for_api, \ voter_merge_two_accounts_action, voter_photo_save_for_api, voter_retrieve_for_api, \ voter_save_photo_from_file_reader, voter_sign_out_for_api, voter_split_into_two_accounts_for_api from voter.controllers_contacts import delete_all_voter_contact_emails_for_voter, save_google_contacts, \ @@ -2095,6 +2096,7 @@ def voter_update_view(request): # voterUpdate is_post = True if request.method == 'POST' else False if is_post: + delete_voter_account = positive_value_exists(request.POST.get('delete_voter_account', False)) facebook_email_changed = positive_value_exists(request.POST.get('facebook_email_changed', False)) facebook_email = request.POST.get('facebook_email', '') if facebook_email_changed else False facebook_profile_image_url_https_changed = \ @@ -2133,6 +2135,7 @@ def voter_update_view(request): # voterUpdate profile_image_type_currently_active_changed = \ positive_value_exists(request.POST.get('profile_image_type_currently_active_changed', False)) else: + delete_voter_account = positive_value_exists(request.GET.get('delete_voter_account', False)) facebook_email, facebook_email_changed = \ return_string_value_and_changed_boolean_from_get(request, 'facebook_email') facebook_profile_image_url_https, facebook_profile_image_url_https_changed = \ @@ -2260,6 +2263,27 @@ def voter_update_view(request): # voterUpdate # At this point, we have a valid voter voter = voter_results['voter'] voter_we_vote_id = voter.we_vote_id + + if delete_voter_account: + # We want to fully delete this record + results = delete_all_voter_information_permanently(voter_to_delete=voter) + if results['success']: + status += "VOTER_DELETED_COMPLETELY " + json_data = { + 'status': status, + 'success': True, + 'voter_deleted': True, + } + else: + status += "VOTER_NOT_DELETED: " + results['status'] + " " + json_data = { + 'status': status, + 'success': True, + 'voter_not_deleted': True, + } + response = HttpResponse(json.dumps(json_data), content_type='application/json') + return response + voter_full_name_at_start = voter.get_full_name(real_name_only=True) if at_least_one_variable_has_changed or external_voter_id_to_be_saved: diff --git a/election/views_admin.py b/election/views_admin.py index 6e17eafc8..50a5aaa6a 100644 --- a/election/views_admin.py +++ b/election/views_admin.py @@ -1048,6 +1048,7 @@ def election_list_view(request): # Upcoming refresh date scheduled? refresh_date_added_to_queue = None + retrieve_date_added_to_queue = None try: batch_process_queryset = BatchProcess.objects.all() batch_process_queryset = \ @@ -1067,6 +1068,9 @@ def election_list_view(request): if one_batch_process.kind_of_process == 'REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS': if not refresh_date_added_to_queue and one_batch_process.date_added_to_queue: refresh_date_added_to_queue = one_batch_process.date_added_to_queue + elif one_batch_process.kind_of_process == 'RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS': + if not retrieve_date_added_to_queue and one_batch_process.date_added_to_queue: + retrieve_date_added_to_queue = one_batch_process.date_added_to_queue except BatchProcess.DoesNotExist: # No offices found. Not a problem. batch_process_list = [] @@ -1079,6 +1083,7 @@ def election_list_view(request): election.refresh_date_added_to_queue = refresh_date_added_to_queue election.retrieve_date_completed = retrieve_date_completed election.retrieve_date_started = retrieve_date_started + election.retrieve_date_added_to_queue = retrieve_date_added_to_queue if refresh_date_completed: most_recent_time = refresh_date_completed diff --git a/email_outbound/models.py b/email_outbound/models.py index b72db0cf5..9c9d5faa1 100644 --- a/email_outbound/models.py +++ b/email_outbound/models.py @@ -3,9 +3,11 @@ # -*- coding: UTF-8 -*- from datetime import date, timedelta -from django.core.mail import EmailMultiAlternatives from django.apps import apps from django.db import models +from sendgrid import SendGridAPIClient +from sendgrid.helpers.mail import * +from config.base import get_environment_variable from wevote_functions.functions import convert_to_int, extract_email_addresses_from_string, generate_random_string, \ positive_value_exists from wevote_settings.models import fetch_next_we_vote_id_email_integer, fetch_site_unique_id_prefix @@ -57,6 +59,8 @@ (SENT, 'Message sent'), ) +SENDGRID_API_KEY = get_environment_variable("SENDGRID_API_KEY", no_exception=True) + class EmailAddress(models.Model): """ @@ -923,6 +927,7 @@ def send_scheduled_email_via_sendgrid(self, email_scheduled): """ status = "" success = True + email_scheduled_sent = False sendgrid_turned_off_for_testing = False if sendgrid_turned_off_for_testing: status += "ERROR_SENDGRID_TURNED_OFF_FOR_TESTING " @@ -934,32 +939,38 @@ def send_scheduled_email_via_sendgrid(self, email_scheduled): } return results - if positive_value_exists(email_scheduled.sender_voter_name): - # TODO DALE Make system variable - system_sender_email_address = "{sender_voter_name} via We Vote " \ - "".format(sender_voter_name=email_scheduled.sender_voter_name) - else: - system_sender_email_address = "We Vote " # TODO DALE Make system variable - - mail = EmailMultiAlternatives( - subject=email_scheduled.subject, - body=email_scheduled.message_text, - from_email=system_sender_email_address, - to=[email_scheduled.recipient_voter_email], - # headers={"Reply-To": email_scheduled.sender_voter_email} - ) - # 2020-01-19 Dale commented out Reply-To header because with it, Gmail gives phishing warning - if positive_value_exists(email_scheduled.message_html): - mail.attach_alternative(email_scheduled.message_html, "text/html") - try: - mail.send() - status += "SENDING_VIA_SENDGRID " - email_scheduled_sent = True + message = Mail() + if positive_value_exists(email_scheduled.sender_voter_name): + message.from_email = From( + 'info@wevote.us', + "{sender_voter_name} via We Vote".format(sender_voter_name=email_scheduled.sender_voter_name)) + else: + message.from_email = From('info@wevote.us', 'We Vote') + message.reply_to = ReplyTo('info@wevote.us', 'We Vote') + message.to = To(email_scheduled.recipient_voter_email, email_scheduled.recipient_voter_email, p=0) + message.subject = Subject(email_scheduled.subject) + message.content = Content( + MimeType.text, + email_scheduled.message_text) + message.content = Content( + MimeType.html, + email_scheduled.message_html) + try: + sendgrid_client = SendGridAPIClient(SENDGRID_API_KEY) + response = sendgrid_client.send(message) + # print(response.status_code) + # print(response.body) + print(response.headers) + status += "SENDING_VIA_SENDGRID " + email_scheduled_sent = True + except Exception as e: + status += "ERROR_COULD_NOT_SEND_VIA_SENDGRID: " + str(e) + ' ' + print(status) + email_scheduled_sent = False except Exception as e: - status += "ERROR_COULD_NOT_SEND_VIA_SENDGRID: " + str(e) + ' ' + status += "ERROR_COULD_NOT_BE_PREPARED_FOR_SENDGRID: " + str(e) + ' ' print(status) - email_scheduled_sent = False results = { 'success': success, diff --git a/requirements.txt b/requirements.txt index f1321dae2..2fea3f727 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,7 +46,8 @@ requests-oauthlib==1.3.0 robot-detection==0.4 rsa==4.7.2 selenium==3.141.0 -sendgrid-django==4.2.0 +sendgrid==6.9.7 +# sendgrid-django==4.2.0 setuptools simplejson==3.17.2 six==1.14.0 diff --git a/templates/election/election_list.html b/templates/election/election_list.html index c044e4ae1..b58428272 100644 --- a/templates/election/election_list.html +++ b/templates/election/election_list.html @@ -211,6 +211,12 @@

{% if is_national_election %}{{ national_election.election_name }} — { start: {{ election.refresh_date_started }} + {% elif election.retrieve_date_completed %} + {{ election.retrieve_date_completed }} + {% elif election.retrieve_date_started %} + + start: {{ election.retrieve_date_started }} + {% endif %} diff --git a/templates/email_outbound/email_templates/friend_invitation.html b/templates/email_outbound/email_templates/friend_invitation.html index 8be3ff87a..58d3f3283 100644 --- a/templates/email_outbound/email_templates/friend_invitation.html +++ b/templates/email_outbound/email_templates/friend_invitation.html @@ -149,13 +149,13 @@   - If the Accept Invitation button does not work, please copy and paste this link into a browser: {{ confirm_friend_request_url }} + If Accept Invitation button does not work, please copy and paste this link into a browser to accept the invitation: {{ confirm_friend_request_url }}   - If the See All Requests button does not work, please use this link: {{ see_all_friend_requests_url }} + If See All Requests button does not work, please use this link: {{ see_all_friend_requests_url }} @@ -179,7 +179,6 @@   - diff --git a/templates/email_outbound/email_templates/sign_in_code_email.html b/templates/email_outbound/email_templates/sign_in_code_email.html index bd3b06e09..250f438bf 100644 --- a/templates/email_outbound/email_templates/sign_in_code_email.html +++ b/templates/email_outbound/email_templates/sign_in_code_email.html @@ -126,7 +126,6 @@   - diff --git a/templates/email_outbound/email_templates/sign_in_code_email.txt b/templates/email_outbound/email_templates/sign_in_code_email.txt index 843a36606..735381d5a 100644 --- a/templates/email_outbound/email_templates/sign_in_code_email.txt +++ b/templates/email_outbound/email_templates/sign_in_code_email.txt @@ -11,6 +11,6 @@ The We Vote Team ======================================== -This message was sent to {{ recipient_email }}. If you don't want to receive emails from We Vote, please follow the link below to unsubscribe: -{{ recipient_unsubscribe }} +This message was sent to {{ recipient_voter_email }}. If you don't want to receive emails from We Vote, please follow the link below to unsubscribe: +{{ recipient_unsubscribe_url }} We Vote, Attention: Community Team, 1423 Broadway PMB 158, Oakland, CA 94612 \ No newline at end of file diff --git a/voter/controllers.py b/voter/controllers.py index 1085b064d..c61319362 100644 --- a/voter/controllers.py +++ b/voter/controllers.py @@ -296,7 +296,7 @@ def delete_all_voter_information_permanently(voter_to_delete=None): # voterDele # And finally, delete all voter_device_links for this voter update_link_results = voter_device_link_manager.delete_all_voter_device_links_by_voter_id(voter_to_delete_id) - if update_link_results['voter_device_link_updated']: + if update_link_results['success']: success = True status += "VOTER_DEVICE_LINK_DELETED " else: From 1f688010a2cdc9e8ac4bd52e62d135ffe6c2253d Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Sun, 10 Jul 2022 10:02:06 -0700 Subject: [PATCH 25/98] Removed some commented out HTML from many email templates (for improved deliverability). Fixed some incorrect variable names in email templates. Added election order by name. --- election/views_admin.py | 5 +++-- .../email_templates/campaignx_friend_has_supported.html | 1 - .../email_outbound/email_templates/campaignx_news_item.html | 1 - .../email_templates/campaignx_super_share_item.html | 1 - .../campaignx_supporter_initial_response.html | 1 - .../email_templates/friend_accepted_invitation.html | 1 - .../email_outbound/email_templates/link_to_sign_in.html | 1 - templates/email_outbound/email_templates/link_to_sign_in.txt | 4 ++-- .../email_outbound/email_templates/message_to_friend.html | 1 - .../email_templates/notice_friend_endorsements.html | 1 - .../email_templates/notice_voter_daily_summary.html | 1 - .../email_templates/send_ballot_to_friends.html | 1 - .../email_outbound/email_templates/send_ballot_to_self.html | 1 - .../email_outbound/email_templates/verify_email_address.html | 1 - .../email_outbound/email_templates/verify_email_address.txt | 4 ++-- 15 files changed, 7 insertions(+), 18 deletions(-) diff --git a/election/views_admin.py b/election/views_admin.py index 50a5aaa6a..ef0669c53 100644 --- a/election/views_admin.py +++ b/election/views_admin.py @@ -864,7 +864,7 @@ def election_list_view(request): office_manager = ContestOfficeManager() election_list_query = Election.objects.all() - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_name', 'election_day_text') election_list_query = election_list_query.exclude(google_civic_election_id=2000) if positive_value_exists(show_ignored_elections): # Do not filter out ignored elections @@ -1056,7 +1056,8 @@ def election_list_view(request): batch_process_queryset = batch_process_queryset.filter(date_completed__isnull=True) batch_process_queryset = batch_process_queryset.exclude(batch_process_paused=True) ballot_item_processes = [ - 'REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS'] + 'REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS', + 'RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS'] batch_process_queryset = batch_process_queryset.filter(kind_of_process__in=ballot_item_processes) batch_process_queryset = batch_process_queryset.order_by("-id") diff --git a/templates/email_outbound/email_templates/campaignx_friend_has_supported.html b/templates/email_outbound/email_templates/campaignx_friend_has_supported.html index a70bc45c0..5d4d3f41c 100644 --- a/templates/email_outbound/email_templates/campaignx_friend_has_supported.html +++ b/templates/email_outbound/email_templates/campaignx_friend_has_supported.html @@ -135,7 +135,6 @@   - diff --git a/templates/email_outbound/email_templates/campaignx_news_item.html b/templates/email_outbound/email_templates/campaignx_news_item.html index 484e4ec70..f9efe151d 100644 --- a/templates/email_outbound/email_templates/campaignx_news_item.html +++ b/templates/email_outbound/email_templates/campaignx_news_item.html @@ -199,7 +199,6 @@   - diff --git a/templates/email_outbound/email_templates/campaignx_super_share_item.html b/templates/email_outbound/email_templates/campaignx_super_share_item.html index 5a8ee2c15..82ea0ea4f 100644 --- a/templates/email_outbound/email_templates/campaignx_super_share_item.html +++ b/templates/email_outbound/email_templates/campaignx_super_share_item.html @@ -195,7 +195,6 @@   - diff --git a/templates/email_outbound/email_templates/campaignx_supporter_initial_response.html b/templates/email_outbound/email_templates/campaignx_supporter_initial_response.html index 55906505d..971eb2f90 100644 --- a/templates/email_outbound/email_templates/campaignx_supporter_initial_response.html +++ b/templates/email_outbound/email_templates/campaignx_supporter_initial_response.html @@ -160,7 +160,6 @@   - diff --git a/templates/email_outbound/email_templates/friend_accepted_invitation.html b/templates/email_outbound/email_templates/friend_accepted_invitation.html index 256b1c9af..c0443e74f 100644 --- a/templates/email_outbound/email_templates/friend_accepted_invitation.html +++ b/templates/email_outbound/email_templates/friend_accepted_invitation.html @@ -169,7 +169,6 @@   - diff --git a/templates/email_outbound/email_templates/link_to_sign_in.html b/templates/email_outbound/email_templates/link_to_sign_in.html index ee2eeb557..9d0e3f061 100644 --- a/templates/email_outbound/email_templates/link_to_sign_in.html +++ b/templates/email_outbound/email_templates/link_to_sign_in.html @@ -128,7 +128,6 @@   - diff --git a/templates/email_outbound/email_templates/link_to_sign_in.txt b/templates/email_outbound/email_templates/link_to_sign_in.txt index 797bca917..9078e4173 100644 --- a/templates/email_outbound/email_templates/link_to_sign_in.txt +++ b/templates/email_outbound/email_templates/link_to_sign_in.txt @@ -11,6 +11,6 @@ The We Vote Team ======================================== -This message was sent to {{ recipient_email }}. If you don't want to receive emails from We Vote, please follow the link below to unsubscribe: -{{ recipient_unsubscribe }} +This message was sent to {{ recipient_voter_email }}. If you don't want to receive emails from We Vote, please follow the link below to unsubscribe: +{{ recipient_unsubscribe_url }} We Vote, Attention: Community Team, 1423 Broadway PMB 158, Oakland, CA 94612 \ No newline at end of file diff --git a/templates/email_outbound/email_templates/message_to_friend.html b/templates/email_outbound/email_templates/message_to_friend.html index 062c223c7..d87f55cd6 100644 --- a/templates/email_outbound/email_templates/message_to_friend.html +++ b/templates/email_outbound/email_templates/message_to_friend.html @@ -166,7 +166,6 @@   - diff --git a/templates/email_outbound/email_templates/notice_friend_endorsements.html b/templates/email_outbound/email_templates/notice_friend_endorsements.html index c2611d8d5..50225726e 100644 --- a/templates/email_outbound/email_templates/notice_friend_endorsements.html +++ b/templates/email_outbound/email_templates/notice_friend_endorsements.html @@ -179,7 +179,6 @@   - diff --git a/templates/email_outbound/email_templates/notice_voter_daily_summary.html b/templates/email_outbound/email_templates/notice_voter_daily_summary.html index ec6dbc7bd..c2945348f 100644 --- a/templates/email_outbound/email_templates/notice_voter_daily_summary.html +++ b/templates/email_outbound/email_templates/notice_voter_daily_summary.html @@ -187,7 +187,6 @@   - diff --git a/templates/email_outbound/email_templates/send_ballot_to_friends.html b/templates/email_outbound/email_templates/send_ballot_to_friends.html index 22c2441d5..8e1f90baa 100644 --- a/templates/email_outbound/email_templates/send_ballot_to_friends.html +++ b/templates/email_outbound/email_templates/send_ballot_to_friends.html @@ -78,7 +78,6 @@   - diff --git a/templates/email_outbound/email_templates/send_ballot_to_self.html b/templates/email_outbound/email_templates/send_ballot_to_self.html index e7ab87fbc..ac44a96a1 100644 --- a/templates/email_outbound/email_templates/send_ballot_to_self.html +++ b/templates/email_outbound/email_templates/send_ballot_to_self.html @@ -81,7 +81,6 @@   - diff --git a/templates/email_outbound/email_templates/verify_email_address.html b/templates/email_outbound/email_templates/verify_email_address.html index 6960a2982..d3ac793b0 100644 --- a/templates/email_outbound/email_templates/verify_email_address.html +++ b/templates/email_outbound/email_templates/verify_email_address.html @@ -127,7 +127,6 @@   - diff --git a/templates/email_outbound/email_templates/verify_email_address.txt b/templates/email_outbound/email_templates/verify_email_address.txt index 58c0a3b56..a9c4b41f7 100644 --- a/templates/email_outbound/email_templates/verify_email_address.txt +++ b/templates/email_outbound/email_templates/verify_email_address.txt @@ -11,6 +11,6 @@ The We Vote Team ======================================== -This message was sent to {{ recipient_email }}. If you don't want to receive emails from We Vote, please follow the link below to unsubscribe: -{{ recipient_unsubscribe }} +This message was sent to {{ recipient_voter_email }}. If you don't want to receive emails from We Vote, please follow the link below to unsubscribe: +{{ recipient_unsubscribe_url }} We Vote, Attention: Community Team, 1423 Broadway PMB 158, Oakland, CA 94612 \ No newline at end of file From c094ae3ae47912bd3432676ca43ca3c17eaacaff Mon Sep 17 00:00:00 2001 From: stevepodell Date: Mon, 11 Jul 2022 15:39:29 -0700 Subject: [PATCH 26/98] Catch an exception that is happening in retrieve_sql_tables_as_csv for the "FAST LOAD ALL OF THE ELECTION DATA, TO YOUR LOCAL POSTGRES" button --- docs/README_MAC_SIMPLIFIED_INSTALL.md | 253 ++++++++++++++------------ requirements.txt | 2 +- retrieve_tables/controllers.py | 51 +++--- 3 files changed, 160 insertions(+), 146 deletions(-) diff --git a/docs/README_MAC_SIMPLIFIED_INSTALL.md b/docs/README_MAC_SIMPLIFIED_INSTALL.md index 4d3e7ccae..cd80a5f1e 100644 --- a/docs/README_MAC_SIMPLIFIED_INSTALL.md +++ b/docs/README_MAC_SIMPLIFIED_INSTALL.md @@ -15,10 +15,10 @@ installed on your Mac), follow these instructions. They should take an hour or 1. Install the Chrome browser for Mac -1. Open the Mac "App Store" app, and download the current version of Apple's Xcode, which includes "c" language compilers +2. Open the Mac "App Store" app, and download the current version of Apple's Xcode, which includes "c" language compilers and native git integration. This download also includes Apple's Xcode IDE for macOS and iOS native development. - **Note: Xcode requires about 30 GB of disk space, if you don't have much that room on your Mac, it is sufficient + **Note: Xcode requires about 13 GB of disk space, if you don't have much that room on your Mac, it is sufficient to download only the "Xcode Command Line Tools". Unfortunately you need to sign up as an Apple developer to do that. Download (the latest version of) "Command Line Tools for Xcode 13" at [https://developer.apple.com/download/more/](https://developer.apple.com/download/more/). These tools only require 185 MB @@ -28,191 +28,202 @@ installed on your Mac), follow these instructions. They should take an hour or the app store: -1. Start xcode (you can find it with Spotlight, or in the Application folder) +3. Start xcode (you can find it with Spotlight, or in the Application folder) -1. When prompted, download the "Additional Components" (the Command Line Tools). This takes many minutes to complete. +4. July 2022, this step happens without a prompt: When prompted, download the "Additional Components" (the Command Line Tools). This takes many minutes to complete. -1. When you get to "Welcome to Xcode", quit out of the app. (For the WeVoteServer, we only need the command line tools that +5. When you get to "Welcome to Xcode", quit out of the app. (For the WeVoteServer, we only need the command line tools that come with Xcode.) -1. Navigate in Chrome to [GitHub](https://GitHub.com). Create a personal account if you don't already have one. +6. Navigate in Chrome to [GitHub](https://GitHub.com). Create a personal account if you don't already have one. -1. Within the GitHub site, navigate to [https://GitHub.com/wevote/WeVoteServer](https://GitHub.com/wevote/WeVoteServer). +7. Within the GitHub site, navigate to [https://GitHub.com/wevote/WeVoteServer](https://GitHub.com/wevote/WeVoteServer). Create a fork of wevote/WeVoteServer.git by selecting the "Fork" button (in the upper right of screen). -1. Download and install the Community version of PyCharm, it's free! +8. Download and install the Community version of PyCharm, it's free! (If you are a student, you can get PyCharem Professional for free. Professional is nice, but not necessary.) [https://www.jetbrains.com/pycharm/download/#section=mac](https://www.jetbrains.com/pycharm/download/#section=mac) -1. StartPyCharm, and press the 'Get from VCS' button. +9. StartPyCharm, and press the 'Get from VCS' button. -1. Clone your fork of the git repository, by copying the URL to the repository into the URL filed, then press the Clone button. +10. Clone your fork of the git repository, by copying the URL to the repository into the URL filed, then press the Clone button. _What this means in english is that you have created a copy in GitHub of the WeVoteServer codebase, and cloning it downloads a copy of your copy to your Mac. At this instant, the 'develop' branch of wevote/WeVoteServer matches - your branch (in this example) SailingSteve/WeVoteServer and also matches the code on your Mac. + your branch (in this example) SailingSteve/WeVoteServer and also matches the code on your Mac. - + -1. The PyCharm IDE appears in 'Dracula' mode, with the repository loaded to your disk, and ready to edit. +11. The PyCharm IDE appears in 'Dracula' mode, with the repository loaded to your disk, and ready to edit. - + -1. If you like 'Dracula' mode, you can skip this step. Open PyCharm/Preferences and press the +12. If you like 'Dracula' mode, you can skip this step. Open PyCharm/Preferences and press the 'Sync with OS' button to match the display mode of your Mac. - - + + -1. In PyCharm/Preferences/Plugins enable the Markdown and IdeaVim tools (this takes a while). +13. In PyCharm/Preferences/Plugins enable the IdeaVim tool (this takes a while). Feel free to add any other PyCharm tools that you would like! When done press 'Ok', and the IDE will reboot. - + + +14. If you are using one of the newer Macs with Apple Silicon processor, he installer offers the "Apple Silicon Version" which is better and more stable -- take it if it is offered! -1. If the Apple top menu, shows "Git" skip this step. If it says "VCS", the follow this step to configure Git +15. If the Apple top menu, shows "Git" skip this step. If it says "VCS", the follow this step to configure Git - + - Select 'Git' on the VCS meu, and press Ok. + Select 'Git' on the VCS meu, and press Ok. - + -1. In PyCharm set your git remotes. Navigate to the Git/'Manage Remotes...' dialog +16. In PyCharm set your git remotes. Navigate to the Git/'Manage Remotes...' dialog (July 2022: these two images might be reversed! (Verify!) But the results in the next step are corect.) - ![ScreenShot](images/RemotesUpstream.png) + ![ScreenShot](images/RemotesUpstream.png) - The WeVoteServer project defines upstream and origin differently than most projects. + The WeVoteServer project defines upstream and origin differently than most projects. - Click the edit (pencil) icon, and change the word origin to upstream. This is how it looks after the change. + Click the edit (pencil) icon, and change the word origin to upstream. This is how it looks after the change. - ![ScreenShot](images/RemotesOrigin.png) + ![ScreenShot](images/RemotesOrigin.png) -1. Then add a remote for your private branch by pressing the '+' button on the Git Remotes dialog. Add the url for your - fork of the WeVoteServer project origin (copy the url from the GitHub website). In this example, the developer - is "SailingSteve". +17. Then add a remote for your private branch by pressing the '+' button on the Git Remotes dialog. Add the url for your + fork of the WeVoteServer project origin (copy the url from the GitHub website). In this example, the developer + is "SailingSteve". - ![ScreenShot](images/AddUpstream2021.png) + ![ScreenShot](images/AddUpstream2021.png) -1. When the cloning is complete, it will look something like this. +18. When the cloning is complete, it will look something like this. - ![ScreenShot](images/CorrectOrigin2021.png) + ![ScreenShot](images/CorrectOrigin2021.png) - Press Ok to close the dialog + Press Ok to close the dialog -1. In PyCharm copy `environment_variables-template.json` to `environment_variables.json` +19. In PyCharm copy `environment_variables-template.json` to `environment_variables.json` - ![ScreenShot](images/PyCharmTemplateCopy2021.png) + ![ScreenShot](images/PyCharmTemplateCopy2021.png) - Right click on `environment_variables-template.json` and select 'Copy', then right click paste on the `config` - directory and select 'Paste' in the pop-up, and then in the copy dialog that open up, and change the "new name:" to - `environment_variables.json` + Right click on `environment_variables-template.json` and select 'Copy', then right click paste on the `config` + directory and select 'Paste' in the pop-up, and then in the copy dialog that open up, and change the "new name:" to + `environment_variables.json` - If you skip this step, in a much later step, when you run "makemigrations", it will fail with an - 'Unable to set the **** variable from "os.environ" or JSON file' error. + If you skip this step, in a much later step, when you run "makemigrations", it will fail with an + 'Unable to set the **** variable from "os.environ" or JSON file' error. - **There are a number of secret values in `environment_variables.json` that are not in source control, - you will need to check in with Dale, as you find that you need them.** + **There are a number of secret values in `environment_variables.json` that are not in source control, + you will need to check in with Dale, as you find that you need them.** -1. In PyCharm, open the Terminal window and accept use of the z shell (if you want to use some other shell, feel free to skip this step). +20. In PyCharm, open the Terminal window and accept use of the z shell (if you want to use some other shell, feel free to skip this step). - ![ScreenShot](images/AcceptZShell.png) + ![ScreenShot](images/AcceptZShell.png) - The terminal opens up with the project root directory set as the pwd (which is handy). + The terminal opens up with the project root directory set as the pwd (which is handy). -1. In the PyCharm terminal window download [Homebrew]( https://brew.sh/) ("the missing package manager for macOS") by entering +21. In the PyCharm terminal window download [Homebrew]( https://brew.sh/) ("the missing package manager for macOS") by entering the following command: - ``` - $ /bin/bash -c "$(curl -fsSL https://raw.GitHubusercontent.com/Homebrew/install/master/install.sh)" - ``` - - This loads and runs a Ruby script (Ruby comes pre-installed in macOS), and Ruby uses curl (also pre-loaded) to pull the file - into the bash (terminal) command shell for execution. This Ruby script also internally uses 'sudo' which temporarily gives - the script root privileges to install software, so you will need to know an admin password for your Mac. - - This script can take a few minutes to complete. - -2. Install the latest version of Python - - ``` - $ brew install python - ``` - If an older version of Python has been installed, and the installation fails, you will see the following error: - ``` - Error: python@3.9 3.9.1_1 is already installed - To upgrade to 3.9.5, run: - brew upgrade python@3.9 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % - ``` - In which case you run the suggested upgrade command, in this example it would be `brew upgrade python@3.9`, then finally export the path as shown below. - ``` - $ export PATH="/usr/local/opt/python/libexec/bin:$PATH" - ``` -3. Test that the newly installed Python is in the path. macOS comes with Python 2 preinstalled, so + ``` + $ /bin/bash -c "$(curl -fsSL https://raw.GitHubusercontent.com/Homebrew/install/master/install.sh)" + ``` + + This loads and runs a Ruby script (Ruby comes pre-installed in macOS), and Ruby uses curl (also pre-loaded) to pull the file + into the bash (terminal) command shell for execution. This Ruby script also internally uses 'sudo' which temporarily gives + the script root privileges to install software, so you will need to know an admin password for your Mac. + + This script can take a few minutes to complete. + +22. Install the latest version of Python + + ``` + $ brew install python + ``` + If an older version of Python has been installed, and the installation fails, you will see the following error: + ``` + Error: python@3.9 3.9.1_1 is already installed + To upgrade to 3.9.5, run: + brew upgrade python@3.9 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % + ``` + In which case you run the suggested upgrade command, in this example it would be `brew upgrade python@3.9`, then finally export the path as shown below. + ``` + $ export PATH="/usr/local/opt/python/libexec/bin:$PATH" + ``` +23. Test that the newly installed Python is in the path. macOS comes with Python 2 preinstalled, so if the reported version is 2, then add the newly loaded python to the path with the export command. Then confirm that the default python is now version 3.9 or later. (Version 3.6 has problems with macOS Big Sur or later) + ``` + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version + Python 2.7.16 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % export PATH="/usr/local/opt/python/libexec/bin:$PATH" + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version + Python 3.9.5 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % + ``` + 2021: For an 'Apple M1 Max' ARM-64 Processor... + ``` + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version + Python 2.7.18 + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % export PATH="/opt/homebrew/opt/python@3.9/libexec/bin:$PATH" + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version + Python 3.9.9 + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % + ``` + Note July 2022: n the homebrew directory (not in the venv), had to make a symlink between python3 and python so that psycopg2-binary could find python. (To be verified) + +24. If python --version fails, + try ``` - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version - Python 2.7.16 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % export PATH="/usr/local/opt/python/libexec/bin:$PATH" - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version - Python 3.9.5 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % - ``` - 2021: For an 'Apple M1 Max' ARM-64 Processor... - ``` - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version - Python 2.7.18 - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % export PATH="/opt/homebrew/opt/python@3.9/libexec/bin:$PATH" - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version - Python 3.9.9 - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % + ln -s /opt/homebrew/bin/python3 /opt/homebrew/bin/python ``` +bneeded to install postgres before the requirements because psyco3-3 binary requires pg_config which is not installed yet. -5. Set up a Virtual Environment with the new Python Interpreter. + +25. Set up a Virtual Environment with the new Python Interpreter. Navigate to: PyCharm/Preferences/Project: WeVoteServer/Python Interpreter. - + -6. Click the Gear icon, then select "Add". PyCharm will detect the latest interpreter from the PATH environment variable, - and pre-populate the dialog. Check the two checkboxes `Inherit global site-packages` and `make available to all projects`. +26. Click the Gear icon, then select "Add". PyCharm will detect the latest interpreter from the PATH environment variable, + and pre-populate the dialog. Check the two checkboxes `Inherit global site-packages` and `make available to all projects`. - + - Confirm that the 'Base interpreter' field shows us using the Python version that you just downloaded, and it knows the location for pip, setuptools, and wheel (3 python utilities). - Then press Ok. + Confirm that the 'Base interpreter' field shows us using the Python version that you just downloaded, and it knows the location for pip, setuptools, and wheel (3 python utilities). + Then press Ok. - ![ScreenShot](images/VenvCompleted.png) + ![ScreenShot](images/VenvCompleted.png) -7. Confirm that the new virtual environment is in effect, by closing all open Terminal windows within +27. Confirm that the new virtual environment is in effect, by closing all open Terminal windows within PyCharm and opening a new one. - + - If you see '(venv)' at the beginning of the command line, all is well. + If you see '(venv)' at the beginning of the command line, all is well. -8. Install OpenSSL, the pyopenssl and https clients: +28. Install OpenSSL, the pyopenssl and https clients: - `(WeVoteServerPy3.7) $ brew install openssl` - If it is already installed, no worries! + `(WeVoteServerPy3.7) $ brew install openssl` + If it is already installed, no worries! -9. Link libssl and libcrypto so that pip can find them: - ``` - $ ln -s /usr/local/opt/openssl/lib/libcrypto.dylib /usr/local/lib/libcrypto.dylib - $ ln -s /usr/local/opt/openssl/lib/libssl.dylib /usr/local/lib/libssl.dylib - ``` -10. Install libmagic +29. Link libssl and libcrypto so that pip can find them: + ``` + $ ln -s /usr/local/opt/openssl/lib/libcrypto.dylib /usr/local/lib/libcrypto.dylib + $ ln -s /usr/local/opt/openssl/lib/libssl.dylib /usr/local/lib/libssl.dylib + ``` +30. Install libmagic `(WeVoteServerPy3.7) $ brew install libmagic` -11. Install all the other Python packages required by the WeVoteServer project (there are a lot of them!) +31. Install all the other Python packages required by the WeVoteServer project (there are a lot of them!) `(WeVoteServer3.7) $ pip3 install -r requirements.txt` @@ -220,6 +231,8 @@ PyCharm and opening a new one. linux/macOS binary libraries based on c language packages and compiled with gcc. Wheels allow python library developers to speed up execution by coding critical or complex sections the c language. Interpreted Python code runs slower than compiled c. + + **Note July 2022 if this fails due to `psycopg2-binary` requiring `pg_config` (which is installed with postgres), install Postgres first then come back and do the pip3 install -r requirements.txt` command.** If this installation succeeds with no missing libraries, or other compiler errors, we are almost done. If this installation fails, please ask for help. @@ -264,7 +277,7 @@ this step. To see if postgres is already running, check with lsof in a terminal `(venv) $ brew services start postgresql` -5. Create a default database, and a default user, and then log into the 'psql postgres' PostgreSQL command interpreter: +5. Create a default database, and a default user, and then log into the 'psql postgres' PostgreSQL command interpreter ("postgres=#" is the command prompt, you should not have to type this in): _New way: November 2021, using Postgres 14.0_ ``` @@ -273,24 +286,22 @@ this step. To see if postgres is already running, check with lsof in a terminal Type "help" for help. postgres=# createdb - postgres-# createuser -s postgres - postgres-# \du + postgres=# createuser -s postgres (TODO 7/8: CREATE ROLE postgres WITH SUPERUSER CREATEDB CREATEROLE LOGIN ENCRYPTED PASSWORD ‘stevePG’; + postgres=# \du List of roles Role name | Attributes | Member of -------------+------------------------------------------------------------+----------- stevepodell | Superuser, Create role, Create DB, Replication, Bypass RLS | {} - postgres-# postgres=# create database WeVoteServerDB; CREATE DATABASE postgres=# grant all privileges on database WeVoteServerDB to postgres; GRANT - postgres=# postgres=# \l List of databases Name | Owner | Encoding | Collate | Ctype | Access privileges ----------------+-------------+----------+---------+-------+----------------------------- - WeVoteServerDB | stevepodell | UTF8 | C | C | + WeVoteServerDB | stevepodell | UTF8 | C | C | (TODO 7/8/22 these don't match the latest setup) postgres | stevepodell | UTF8 | C | C | template0 | stevepodell | UTF8 | C | C | =c/stevepodell + | | | | | stevepodell=CTc/stevepodell @@ -344,10 +355,10 @@ this step. To see if postgres is already running, check with lsof in a terminal This can take a few minutes to complete. When `brew install --cask pgadmin4` finishes, it prints out `Moving App 'pgAdmin 4.app' to '/Applications/pgAdmin 4.app'.` The latest pgAdmin4 has a webapp architecture (it is not a compiled program). The app you start from the Application folder is actually a - single purpose web server, and the UI for the app appears in Chrome as a local website. + single purpose web server, and the UI for the app appears. (7/8/22 needs a new ) 7. Use Spotlight to find and launch the pgAdmin4 app. Once launched, the pgAdmin4 webapp will display in a new tab within Chrome. - On that new tab, Right-click on "Servers" and choose "Create > Server" + On that new tab, click on the "Add new Servers" button and choose "Create > Server" @@ -360,7 +371,7 @@ this step. To see if postgres is already running, check with lsof in a terminal * Port: 5432 * Maintenance database: postgres * User name: postgres - * Password: + * Password: (mine is stevePG) * Save password: checked ![ScreenShot](images/CreateServerConnection2.png) @@ -462,7 +473,7 @@ this step. To see if postgres is already running, check with lsof in a terminal (WeVoteServer3.7) admin$ ``` -1. Navigate to [http://localhost:8000/admin/](http://localhost:8000/admin/) and sign in with your new username/password. +1. Navigate to [http://localhost:8000/admin/](http://localhost:8000/admin/) and sign in with your new username/password (for example mine is stevepodell/stevePG.). 1. Your local instance of the WeVoteServer is now setup and running (although there is no election data stored in your Postgres instance, for it to serve to clients at this point). diff --git a/requirements.txt b/requirements.txt index 2fea3f727..1bce06115 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ oauthlib==3.1.1 nameparser==1.0.6 phonenumbers==8.12.39 Pillow==9.0.1 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 py3dns==3.2.1 diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 9912384aa..a0c8d05f3 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -82,30 +82,33 @@ def retrieve_sql_tables_as_csv(table_name, start, end): csv_files = {} if table_name in allowable_tables: - cur = conn.cursor() - csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') - print("exporting to: " + csv_name) - with open(csv_name, 'w') as file: - if positive_value_exists(end): - sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ - " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - else: - sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - cur.copy_expert(sql, file, size=8192) - logger.error("retrieve_tables sql: " + sql) - file.close() - with open(csv_name, 'r') as file2: - csv_files[table_name] = file2.read() - file2.close() - os.remove(csv_name) - if "exported" not in status: - status += "exported " - status += table_name + "(" + start + "," + end + "), " - conn.commit() - conn.close() - dt = time.time() - t0 - logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + - ' seconds. start = ' + start + ', end = ' + end) + try: + cur = conn.cursor() + csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') + print("exporting to: " + csv_name) + with open(csv_name, 'w') as file: + if positive_value_exists(end): + sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + else: + sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + cur.copy_expert(sql, file, size=8192) + logger.error("retrieve_tables sql: " + sql) + file.close() + with open(csv_name, 'r') as file2: + csv_files[table_name] = file2.read() + file2.close() + os.remove(csv_name) + if "exported" not in status: + status += "exported " + status += table_name + "(" + start + "," + end + "), " + conn.commit() + conn.close() + dt = time.time() - t0 + logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + + ' seconds. start = ' + start + ', end = ' + end) + except Exception as e: + logger.error("Real exception in retrieve_sql_tables_as_csv(): " + str(e) + " ") else: status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) From 83e820b3798fdba594b060ba389f78e19af87400 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 12 Jul 2022 08:53:43 -0700 Subject: [PATCH 27/98] Git date experiment 7 --- templates/admin_tools/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index c1bce4e8a..a41150020 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -172,7 +172,7 @@

Technical Information

{{ postgres_version }} - Git date: (experiment 6)  + Git date: (experiment 7)  {# {{git_merge_date}}#} {% for line in git_merge_date %}

{{ line.date }} --- {{ line.file }}

From 0d25b473d7d5edd72102d2a212e71b75a11a37c8 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 12 Jul 2022 09:00:38 -0700 Subject: [PATCH 28/98] Git date experiment 7 --- retrieve_tables/controllers.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 9912384aa..2769e9c10 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -93,15 +93,21 @@ def retrieve_sql_tables_as_csv(table_name, start, end): sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" cur.copy_expert(sql, file, size=8192) logger.error("retrieve_tables sql: " + sql) + logger.error("experiment 7: " + sql) file.close() + logger.error("experiment 7 after file close ") with open(csv_name, 'r') as file2: csv_files[table_name] = file2.read() file2.close() + logger.error("experiment 7 after second file close ") os.remove(csv_name) + logger.error("experiment 7 after remove ") if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " + logger.error("experiment 7 before conn.commit") conn.commit() + logger.error("experiment 7 after conn.commit ") conn.close() dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + @@ -110,12 +116,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) + logger.error("experiment 7 before results") results = { 'success': True, 'status': status, 'files': csv_files, } + logger.error("experiment 7 results: ", results) return results except Exception as e: @@ -181,7 +189,7 @@ def save_off_database(): time.sleep(20) -def retrieve_sql_files_from_master_server(request): +def retrieve_sql_files_from_master_server(): """ Get the json data, and create new entries in the developers local database :return: From ade0ac3502ea11df07acd939e530748ee97471c0 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 12 Jul 2022 09:07:28 -0700 Subject: [PATCH 29/98] Git date experiment 7 --- docs/README_MAC_SIMPLIFIED_INSTALL.md | 246 +++++++++++++------------- requirements.txt | 2 +- retrieve_tables/controllers.py | 64 +++---- 3 files changed, 161 insertions(+), 151 deletions(-) diff --git a/docs/README_MAC_SIMPLIFIED_INSTALL.md b/docs/README_MAC_SIMPLIFIED_INSTALL.md index 4d3e7ccae..b9fec27e2 100644 --- a/docs/README_MAC_SIMPLIFIED_INSTALL.md +++ b/docs/README_MAC_SIMPLIFIED_INSTALL.md @@ -15,7 +15,7 @@ installed on your Mac), follow these instructions. They should take an hour or 1. Install the Chrome browser for Mac -1. Open the Mac "App Store" app, and download the current version of Apple's Xcode, which includes "c" language compilers +2. Open the Mac "App Store" app, and download the current version of Apple's Xcode, which includes "c" language compilers and native git integration. This download also includes Apple's Xcode IDE for macOS and iOS native development. **Note: Xcode requires about 30 GB of disk space, if you don't have much that room on your Mac, it is sufficient @@ -28,191 +28,199 @@ installed on your Mac), follow these instructions. They should take an hour or the app store: -1. Start xcode (you can find it with Spotlight, or in the Application folder) +3. Start xcode (you can find it with Spotlight, or in the Application folder) -1. When prompted, download the "Additional Components" (the Command Line Tools). This takes many minutes to complete. +4. When prompted, download the "Additional Components" (the Command Line Tools). This takes many minutes to complete. -1. When you get to "Welcome to Xcode", quit out of the app. (For the WeVoteServer, we only need the command line tools that +5. When you get to "Welcome to Xcode", quit out of the app. (For the WeVoteServer, we only need the command line tools that come with Xcode.) -1. Navigate in Chrome to [GitHub](https://GitHub.com). Create a personal account if you don't already have one. +6. Navigate in Chrome to [GitHub](https://GitHub.com). Create a personal account if you don't already have one. -1. Within the GitHub site, navigate to [https://GitHub.com/wevote/WeVoteServer](https://GitHub.com/wevote/WeVoteServer). +7. Within the GitHub site, navigate to [https://GitHub.com/wevote/WeVoteServer](https://GitHub.com/wevote/WeVoteServer). Create a fork of wevote/WeVoteServer.git by selecting the "Fork" button (in the upper right of screen). -1. Download and install the Community version of PyCharm, it's free! +8. Download and install the Community version of PyCharm, it's free! [https://www.jetbrains.com/pycharm/download/#section=mac](https://www.jetbrains.com/pycharm/download/#section=mac) -1. StartPyCharm, and press the 'Get from VCS' button. +9. StartPyCharm, and press the 'Get from VCS' button. -1. Clone your fork of the git repository, by copying the URL to the repository into the URL filed, then press the Clone button. +10. Clone your fork of the git repository, by copying the URL to the repository into the URL filed, then press the Clone button. _What this means in english is that you have created a copy in GitHub of the WeVoteServer codebase, and cloning it downloads a copy of your copy to your Mac. At this instant, the 'develop' branch of wevote/WeVoteServer matches - your branch (in this example) SailingSteve/WeVoteServer and also matches the code on your Mac. + your branch (in this example) SailingSteve/WeVoteServer and also matches the code on your Mac. - + -1. The PyCharm IDE appears in 'Dracula' mode, with the repository loaded to your disk, and ready to edit. +11. The PyCharm IDE appears in 'Dracula' mode, with the repository loaded to your disk, and ready to edit. - + -1. If you like 'Dracula' mode, you can skip this step. Open PyCharm/Preferences and press the +12. If you like 'Dracula' mode, you can skip this step. Open PyCharm/Preferences and press the 'Sync with OS' button to match the display mode of your Mac. - - + + -1. In PyCharm/Preferences/Plugins enable the Markdown and IdeaVim tools (this takes a while). +13. In PyCharm/Preferences/Plugins enable the Markdown and IdeaVim tools (this takes a while). Feel free to add any other PyCharm tools that you would like! When done press 'Ok', and the IDE will reboot. - + -1. If the Apple top menu, shows "Git" skip this step. If it says "VCS", the follow this step to configure Git +14. If the Apple top menu, shows "Git" skip this step. If it says "VCS", the follow this step to configure Git - + - Select 'Git' on the VCS meu, and press Ok. + Select 'Git' on the VCS meu, and press Ok. - + -1. In PyCharm set your git remotes. Navigate to the Git/'Manage Remotes...' dialog +15. In PyCharm set your git remotes. Navigate to the Git/'Manage Remotes...' dialog - ![ScreenShot](images/RemotesUpstream.png) + ![ScreenShot](images/RemotesUpstream.png) - The WeVoteServer project defines upstream and origin differently than most projects. + The WeVoteServer project defines upstream and origin differently than most projects. - Click the edit (pencil) icon, and change the word origin to upstream. This is how it looks after the change. + Click the edit (pencil) icon, and change the word origin to upstream. This is how it looks after the change. - ![ScreenShot](images/RemotesOrigin.png) + ![ScreenShot](images/RemotesOrigin.png) -1. Then add a remote for your private branch by pressing the '+' button on the Git Remotes dialog. Add the url for your - fork of the WeVoteServer project origin (copy the url from the GitHub website). In this example, the developer - is "SailingSteve". +16. Then add a remote for your private branch by pressing the '+' button on the Git Remotes dialog. Add the url for your + fork of the WeVoteServer project origin (copy the url from the GitHub website). In this example, the developer + is "SailingSteve". - ![ScreenShot](images/AddUpstream2021.png) + ![ScreenShot](images/AddUpstream2021.png) -1. When the cloning is complete, it will look something like this. +17. When the cloning is complete, it will look something like this. - ![ScreenShot](images/CorrectOrigin2021.png) + ![ScreenShot](images/CorrectOrigin2021.png) - Press Ok to close the dialog + Press Ok to close the dialog -1. In PyCharm copy `environment_variables-template.json` to `environment_variables.json` +18. In PyCharm copy `environment_variables-template.json` to `environment_variables.json` - ![ScreenShot](images/PyCharmTemplateCopy2021.png) + ![ScreenShot](images/PyCharmTemplateCopy2021.png) - Right click on `environment_variables-template.json` and select 'Copy', then right click paste on the `config` - directory and select 'Paste' in the pop-up, and then in the copy dialog that open up, and change the "new name:" to - `environment_variables.json` + Right click on `environment_variables-template.json` and select 'Copy', then right click paste on the `config` + directory and select 'Paste' in the pop-up, and then in the copy dialog that open up, and change the "new name:" to + `environment_variables.json` - If you skip this step, in a much later step, when you run "makemigrations", it will fail with an - 'Unable to set the **** variable from "os.environ" or JSON file' error. + If you skip this step, in a much later step, when you run "makemigrations", it will fail with an + 'Unable to set the **** variable from "os.environ" or JSON file' error. - **There are a number of secret values in `environment_variables.json` that are not in source control, - you will need to check in with Dale, as you find that you need them.** + **There are a number of secret values in `environment_variables.json` that are not in source control, + you will need to check in with Dale, as you find that you need them.** -1. In PyCharm, open the Terminal window and accept use of the z shell (if you want to use some other shell, feel free to skip this step). +19. In PyCharm, open the Terminal window and accept use of the z shell (if you want to use some other shell, feel free to skip this step). - ![ScreenShot](images/AcceptZShell.png) + ![ScreenShot](images/AcceptZShell.png) - The terminal opens up with the project root directory set as the pwd (which is handy). + The terminal opens up with the project root directory set as the pwd (which is handy). -1. In the PyCharm terminal window download [Homebrew]( https://brew.sh/) ("the missing package manager for macOS") by entering +20. In the PyCharm terminal window download [Homebrew]( https://brew.sh/) ("the missing package manager for macOS") by entering the following command: - ``` - $ /bin/bash -c "$(curl -fsSL https://raw.GitHubusercontent.com/Homebrew/install/master/install.sh)" - ``` - - This loads and runs a Ruby script (Ruby comes pre-installed in macOS), and Ruby uses curl (also pre-loaded) to pull the file - into the bash (terminal) command shell for execution. This Ruby script also internally uses 'sudo' which temporarily gives - the script root privileges to install software, so you will need to know an admin password for your Mac. - - This script can take a few minutes to complete. - -2. Install the latest version of Python - - ``` - $ brew install python - ``` - If an older version of Python has been installed, and the installation fails, you will see the following error: - ``` - Error: python@3.9 3.9.1_1 is already installed - To upgrade to 3.9.5, run: - brew upgrade python@3.9 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % - ``` - In which case you run the suggested upgrade command, in this example it would be `brew upgrade python@3.9`, then finally export the path as shown below. - ``` - $ export PATH="/usr/local/opt/python/libexec/bin:$PATH" - ``` -3. Test that the newly installed Python is in the path. macOS comes with Python 2 preinstalled, so + ``` + $ /bin/bash -c "$(curl -fsSL https://raw.GitHubusercontent.com/Homebrew/install/master/install.sh)" + ``` + + This loads and runs a Ruby script (Ruby comes pre-installed in macOS), and Ruby uses curl (also pre-loaded) to pull the file + into the bash (terminal) command shell for execution. This Ruby script also internally uses 'sudo' which temporarily gives + the script root privileges to install software, so you will need to know an admin password for your Mac. + + This script can take a few minutes to complete. + +21. Install the latest version of Python + + ``` + $ brew install python + ``` + If an older version of Python has been installed, and the installation fails, you will see the following error: + ``` + Error: python@3.9 3.9.1_1 is already installed + To upgrade to 3.9.5, run: + brew upgrade python@3.9 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % + ``` + In which case you run the suggested upgrade command, in this example it would be `brew upgrade python@3.9`, then finally export the path as shown below. + ``` + $ export PATH="/usr/local/opt/python/libexec/bin:$PATH" + ``` +22. Test that the newly installed Python is in the path. macOS comes with Python 2 preinstalled, so if the reported version is 2, then add the newly loaded python to the path with the export command. Then confirm that the default python is now version 3.9 or later. (Version 3.6 has problems with macOS Big Sur or later) + ``` + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version + Python 2.7.16 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % export PATH="/usr/local/opt/python/libexec/bin:$PATH" + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version + Python 3.9.5 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % + ``` + 2021: For an 'Apple M1 Max' ARM-64 Processor... + ``` + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version + Python 2.7.18 + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % export PATH="/opt/homebrew/opt/python@3.9/libexec/bin:$PATH" + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version + Python 3.9.9 + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % + ``` + +23. If python --version fails, + try ``` - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version - Python 2.7.16 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % export PATH="/usr/local/opt/python/libexec/bin:$PATH" - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version - Python 3.9.5 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % - ``` - 2021: For an 'Apple M1 Max' ARM-64 Processor... - ``` - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version - Python 2.7.18 - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % export PATH="/opt/homebrew/opt/python@3.9/libexec/bin:$PATH" - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version - Python 3.9.9 - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % + ln -s /opt/homebrew/bin/python3 /opt/homebrew/bin/python ``` +bneeded to install postgres before the requirements because psyco3-3 binary requires pg_config which is not installed yet. + -5. Set up a Virtual Environment with the new Python Interpreter. +25. Set up a Virtual Environment with the new Python Interpreter. Navigate to: PyCharm/Preferences/Project: WeVoteServer/Python Interpreter. - + -6. Click the Gear icon, then select "Add". PyCharm will detect the latest interpreter from the PATH environment variable, - and pre-populate the dialog. Check the two checkboxes `Inherit global site-packages` and `make available to all projects`. +26. Click the Gear icon, then select "Add". PyCharm will detect the latest interpreter from the PATH environment variable, + and pre-populate the dialog. Check the two checkboxes `Inherit global site-packages` and `make available to all projects`. - + - Confirm that the 'Base interpreter' field shows us using the Python version that you just downloaded, and it knows the location for pip, setuptools, and wheel (3 python utilities). - Then press Ok. + Confirm that the 'Base interpreter' field shows us using the Python version that you just downloaded, and it knows the location for pip, setuptools, and wheel (3 python utilities). + Then press Ok. - ![ScreenShot](images/VenvCompleted.png) + ![ScreenShot](images/VenvCompleted.png) -7. Confirm that the new virtual environment is in effect, by closing all open Terminal windows within +27. Confirm that the new virtual environment is in effect, by closing all open Terminal windows within PyCharm and opening a new one. - + - If you see '(venv)' at the beginning of the command line, all is well. + If you see '(venv)' at the beginning of the command line, all is well. -8. Install OpenSSL, the pyopenssl and https clients: +28. Install OpenSSL, the pyopenssl and https clients: - `(WeVoteServerPy3.7) $ brew install openssl` - If it is already installed, no worries! + `(WeVoteServerPy3.7) $ brew install openssl` + If it is already installed, no worries! -9. Link libssl and libcrypto so that pip can find them: - ``` - $ ln -s /usr/local/opt/openssl/lib/libcrypto.dylib /usr/local/lib/libcrypto.dylib - $ ln -s /usr/local/opt/openssl/lib/libssl.dylib /usr/local/lib/libssl.dylib - ``` -10. Install libmagic +29. Link libssl and libcrypto so that pip can find them: + ``` + $ ln -s /usr/local/opt/openssl/lib/libcrypto.dylib /usr/local/lib/libcrypto.dylib + $ ln -s /usr/local/opt/openssl/lib/libssl.dylib /usr/local/lib/libssl.dylib + ``` +30. Install libmagic `(WeVoteServerPy3.7) $ brew install libmagic` -11. Install all the other Python packages required by the WeVoteServer project (there are a lot of them!) +31. Install all the other Python packages required by the WeVoteServer project (there are a lot of them!) `(WeVoteServer3.7) $ pip3 install -r requirements.txt` @@ -264,7 +272,7 @@ this step. To see if postgres is already running, check with lsof in a terminal `(venv) $ brew services start postgresql` -5. Create a default database, and a default user, and then log into the 'psql postgres' PostgreSQL command interpreter: +5. Create a default database, and a default user, and then log into the 'psql postgres' PostgreSQL command interpreter ("postgres=#" is the command prompt, you should not have to type this in): _New way: November 2021, using Postgres 14.0_ ``` @@ -273,24 +281,22 @@ this step. To see if postgres is already running, check with lsof in a terminal Type "help" for help. postgres=# createdb - postgres-# createuser -s postgres - postgres-# \du + postgres=# createuser -s postgres (TODO 7/8: CREATE ROLE postgres WITH SUPERUSER CREATEDB CREATEROLE LOGIN ENCRYPTED PASSWORD ‘stevePG’; + postgres=# \du List of roles Role name | Attributes | Member of -------------+------------------------------------------------------------+----------- stevepodell | Superuser, Create role, Create DB, Replication, Bypass RLS | {} - postgres-# postgres=# create database WeVoteServerDB; CREATE DATABASE postgres=# grant all privileges on database WeVoteServerDB to postgres; GRANT - postgres=# postgres=# \l List of databases Name | Owner | Encoding | Collate | Ctype | Access privileges ----------------+-------------+----------+---------+-------+----------------------------- - WeVoteServerDB | stevepodell | UTF8 | C | C | + WeVoteServerDB | stevepodell | UTF8 | C | C | (TODO 7/8/22 these don't match the latest setup) postgres | stevepodell | UTF8 | C | C | template0 | stevepodell | UTF8 | C | C | =c/stevepodell + | | | | | stevepodell=CTc/stevepodell @@ -344,10 +350,10 @@ this step. To see if postgres is already running, check with lsof in a terminal This can take a few minutes to complete. When `brew install --cask pgadmin4` finishes, it prints out `Moving App 'pgAdmin 4.app' to '/Applications/pgAdmin 4.app'.` The latest pgAdmin4 has a webapp architecture (it is not a compiled program). The app you start from the Application folder is actually a - single purpose web server, and the UI for the app appears in Chrome as a local website. + single purpose web server, and the UI for the app appears. (7/8/22 needs a new ) 7. Use Spotlight to find and launch the pgAdmin4 app. Once launched, the pgAdmin4 webapp will display in a new tab within Chrome. - On that new tab, Right-click on "Servers" and choose "Create > Server" + On that new tab, click on the "Add new Servers" button and choose "Create > Server" @@ -360,7 +366,7 @@ this step. To see if postgres is already running, check with lsof in a terminal * Port: 5432 * Maintenance database: postgres * User name: postgres - * Password: + * Password: (mine is stevePG) * Save password: checked ![ScreenShot](images/CreateServerConnection2.png) @@ -462,7 +468,7 @@ this step. To see if postgres is already running, check with lsof in a terminal (WeVoteServer3.7) admin$ ``` -1. Navigate to [http://localhost:8000/admin/](http://localhost:8000/admin/) and sign in with your new username/password. +1. Navigate to [http://localhost:8000/admin/](http://localhost:8000/admin/) and sign in with your new username/password (for example mine is stevepodell/stevePG.). 1. Your local instance of the WeVoteServer is now setup and running (although there is no election data stored in your Postgres instance, for it to serve to clients at this point). diff --git a/requirements.txt b/requirements.txt index 2fea3f727..1bce06115 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ oauthlib==3.1.1 nameparser==1.0.6 phonenumbers==8.12.39 Pillow==9.0.1 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 py3dns==3.2.1 diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 2769e9c10..cc0495f9d 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -82,36 +82,40 @@ def retrieve_sql_tables_as_csv(table_name, start, end): csv_files = {} if table_name in allowable_tables: - cur = conn.cursor() - csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') - print("exporting to: " + csv_name) - with open(csv_name, 'w') as file: - if positive_value_exists(end): - sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ - " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - else: - sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - cur.copy_expert(sql, file, size=8192) - logger.error("retrieve_tables sql: " + sql) + try: + cur = conn.cursor() + csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') + print("exporting to: " + csv_name) + with open(csv_name, 'w') as file: + if positive_value_exists(end): + sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + else: + sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 7 retrieve_tables sql: " + sql) logger.error("experiment 7: " + sql) - file.close() - logger.error("experiment 7 after file close ") - with open(csv_name, 'r') as file2: - csv_files[table_name] = file2.read() - file2.close() - logger.error("experiment 7 after second file close ") - os.remove(csv_name) - logger.error("experiment 7 after remove ") - if "exported" not in status: - status += "exported " - status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 7 before conn.commit") - conn.commit() - logger.error("experiment 7 after conn.commit ") - conn.close() - dt = time.time() - t0 - logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + - ' seconds. start = ' + start + ', end = ' + end) + file.close() + logger.error("experiment 7 after file close ") + with open(csv_name, 'r') as file2: + csv_files[table_name] = file2.read() + file2.close() + logger.error("experiment 7 after second file close ") + os.remove(csv_name) + logger.error("experiment 7 after remove ") + if "exported" not in status: + status += "exported " + status += table_name + "(" + start + "," + end + "), " + logger.error("experiment 7 before conn.commit") + conn.commit() + logger.error("experiment 7 after conn.commit ") + conn.close() + logger.error("experiment 7 after conn.commit ") + dt = time.time() - t0 + logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + + ' seconds. start = ' + start + ', end = ' + end) + except Exception as e: + logger.error("Real exception in retrieve_sql_tables_as_csv(): " + str(e) + " ") else: status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) @@ -189,7 +193,7 @@ def save_off_database(): time.sleep(20) -def retrieve_sql_files_from_master_server(): +def retrieve_sql_files_from_master_server(request): """ Get the json data, and create new entries in the developers local database :return: From 2eb48ac519addc9be7121ad15678856f666c3ba9 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 12 Jul 2022 11:37:15 -0700 Subject: [PATCH 30/98] Git date experiment 8, new git_commit_hash reader and use /tmp/ as cwd for retrieveSQLTables --- admin_tools/views.py | 3 ++- config/base.py | 5 +++++ retrieve_tables/controllers.py | 22 +++++++++++----------- templates/admin_tools/index.html | 6 +++++- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/admin_tools/views.py b/admin_tools/views.py index 01d60bf9c..feaeb6831 100644 --- a/admin_tools/views.py +++ b/admin_tools/views.py @@ -3,7 +3,7 @@ # -*- coding: UTF-8 -*- from config.base import get_environment_variable, get_git_merge_date, get_node_version, get_postgres_version, \ - get_python_version, LOGIN_URL + get_python_version, LOGIN_URL, get_git_commit_hash from ballot.models import BallotReturned, VoterBallotSaved from candidate.models import CandidateCampaign, CandidateManager from candidate.controllers import candidates_import_from_sample_file @@ -114,6 +114,7 @@ def admin_home_view(request): 'python_version': get_python_version(), 'node_version': get_node_version(), 'git_merge_date': get_git_merge_date(), + 'git_commit_hash': get_git_commit_hash(), 'postgres_version': get_postgres_version(), 'shared_link_clicked_unique_sharer_count': shared_link_clicked_unique_sharer_count, 'shared_link_clicked_unique_viewer_count': shared_link_clicked_unique_viewer_count, diff --git a/config/base.py b/config/base.py index 25c162b18..9487b998a 100644 --- a/config/base.py +++ b/config/base.py @@ -141,6 +141,11 @@ def get_git_merge_date(): return merge_date2 +def get_git_commit_hash(): + file1 = open('git_commit_hash', 'r') + lines = file1.readlines() + return lines + def get_postgres_version(): formatted = 'fail' try: diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index cc0495f9d..446ca4ffc 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -55,7 +55,7 @@ ] dummy_unique_id = 10000000 -LOCAL_TMP_PATH = get_environment_variable('PATH_FOR_TEMP_FILES') or '.' +LOCAL_TMP_PATH = get_environment_variable('PATH_FOR_TEMP_FILES') or '/tmp/' def retrieve_sql_tables_as_csv(table_name, start, end): @@ -93,24 +93,24 @@ def retrieve_sql_tables_as_csv(table_name, start, end): else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" cur.copy_expert(sql, file, size=8192) - logger.error("experiment 7 retrieve_tables sql: " + sql) - logger.error("experiment 7: " + sql) + logger.error("experiment 8 retrieve_tables sql: " + sql) + logger.error("experiment 8: " + sql) file.close() - logger.error("experiment 7 after file close ") + logger.error("experiment 8 after file close ") with open(csv_name, 'r') as file2: csv_files[table_name] = file2.read() file2.close() - logger.error("experiment 7 after second file close ") + logger.error("experiment 8 after second file close ") os.remove(csv_name) - logger.error("experiment 7 after remove ") + logger.error("experiment 8 after remove ") if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 7 before conn.commit") + logger.error("experiment 8 before conn.commit") conn.commit() - logger.error("experiment 7 after conn.commit ") + logger.error("experiment 8 after conn.commit ") conn.close() - logger.error("experiment 7 after conn.commit ") + logger.error("experiment 8 after conn.commit ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -120,14 +120,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 7 before results") + logger.error("experiment 8 before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 7 results: ", results) + logger.error("experiment 8 results: ", results) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index a41150020..7d3da5707 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -172,13 +172,17 @@

Technical Information

{{ postgres_version }} - Git date: (experiment 7)  + Git date: (experiment 8)  {# {{git_merge_date}}#} {% for line in git_merge_date %}

{{ line.date }} --- {{ line.file }}

{% endfor %} + + Git commit hash: (experiment 8)  + {{git_commit_hash}} +

From dcc9253cdc25c4c002a5cba8f81ff46c4e616c11 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Tue, 12 Jul 2022 14:20:00 -0700 Subject: [PATCH 31/98] 3rd try to use /tmp/ as cwd for retrieveSQLTables --- admin_tools/views.py | 6 ++-- config/base.py | 51 +++++++------------------------- retrieve_tables/controllers.py | 2 +- templates/admin_tools/index.html | 19 ++++++------ 4 files changed, 23 insertions(+), 55 deletions(-) diff --git a/admin_tools/views.py b/admin_tools/views.py index feaeb6831..5a95851f3 100644 --- a/admin_tools/views.py +++ b/admin_tools/views.py @@ -2,7 +2,7 @@ # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- -from config.base import get_environment_variable, get_git_merge_date, get_node_version, get_postgres_version, \ +from config.base import get_environment_variable, get_node_version, get_postgres_version, \ get_python_version, LOGIN_URL, get_git_commit_hash from ballot.models import BallotReturned, VoterBallotSaved from candidate.models import CandidateCampaign, CandidateManager @@ -113,8 +113,8 @@ def admin_home_view(request): 'google_civic_election_id': google_civic_election_id, 'python_version': get_python_version(), 'node_version': get_node_version(), - 'git_merge_date': get_git_merge_date(), - 'git_commit_hash': get_git_commit_hash(), + 'git_commit_hash': get_git_commit_hash(False), + 'git_commit_hash_url': get_git_commit_hash(True), 'postgres_version': get_postgres_version(), 'shared_link_clicked_unique_sharer_count': shared_link_clicked_unique_sharer_count, 'shared_link_clicked_unique_viewer_count': shared_link_clicked_unique_viewer_count, diff --git a/config/base.py b/config/base.py index 9487b998a..cac5f0aad 100644 --- a/config/base.py +++ b/config/base.py @@ -104,47 +104,16 @@ def get_node_version(): print('Node version: ' + version) # Something like 'v14.15.1' return version - -def get_git_merge_date(): - # Assume the latest source file has a timestamp that is the git merge date - pattern = '""gm' - list_of_files = [fn for fn in glob.glob('./*/**') - if not os.path.basename(fn).endswith(('/', '_')) and - re.search(r"/+.*?\..*?$", fn)] # exclude directories entries - latest_file_string = max(list_of_files, key=os.path.getctime) - posix_filepath = pathlib.Path(latest_file_string) - stat_of_file = posix_filepath.stat() - git_merge_date = str(datetime.datetime.fromtimestamp(stat_of_file.st_atime)).split('.', 1)[0] + ' ' - git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_mtime)).split('.', 1)[0] + ' ' - git_merge_date += str(datetime.datetime.fromtimestamp(stat_of_file.st_ctime)).split('.', 1)[0] # was mtime here before experiment - merge_date = [] - for fi in list_of_files: # TODO: Test code, remove asap, 6/14/22 - posix_filepath = pathlib.Path(fi) - stat_of_file = posix_filepath.stat() - ts = int(stat_of_file.st_ctime) - tm = datetime.datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') - line = { - 'date': tm, - 'file': fi, - } - merge_date.append(line) - - # out_string = git_merge_date + ' (' + latest_file_string + ')' - # print(out_string) - line = { - 'date': git_merge_date, - 'file': latest_file_string, - } - merge_date2 = sorted(merge_date, key=lambda x: x['date'], reverse=True) - - merge_date2.insert(0, line) - return merge_date2 - - -def get_git_commit_hash(): - file1 = open('git_commit_hash', 'r') - lines = file1.readlines() - return lines +def get_git_commit_hash(full): + try: + file1 = open('git_commit_hash', 'r') + hash = file1.readline().strip() + except: + hash = 'git_commit_hash-file-not-found' + + if full: + return "https://github.com/wevote/WeVoteServer/pull/1862/commits/" + hash + return hash def get_postgres_version(): formatted = 'fail' diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 446ca4ffc..0c6f74a87 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -55,7 +55,7 @@ ] dummy_unique_id = 10000000 -LOCAL_TMP_PATH = get_environment_variable('PATH_FOR_TEMP_FILES') or '/tmp/' +LOCAL_TMP_PATH = '/tmp/' def retrieve_sql_tables_as_csv(table_name, start, end): diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index 7d3da5707..7e6cb96b1 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -5,6 +5,11 @@ {% block content %} {% load humanize %} +

We Vote Admin Menu

{% if user and not user.is_anonymous %}

@@ -172,16 +177,10 @@

Technical Information

{{ postgres_version }} - Git date: (experiment 8)  -{# {{git_merge_date}}#} - {% for line in git_merge_date %} -

{{ line.date }} --- {{ line.file }}

- {% endfor %} - - - - Git commit hash: (experiment 8)  - {{git_commit_hash}} + Git commit hash: + + {{git_commit_hash}} +

From a84cc9caa0b1167bacd1c2674209b99ad4531b83 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 08:46:30 -0700 Subject: [PATCH 32/98] Experiment 10, still confirming /tmp/ dir for retrieve_sql_tables_as_csv --- retrieve_tables/controllers.py | 24 +++++++++++++----------- templates/admin_tools/index.html | 4 ++++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 0c6f74a87..1ac2fdd5f 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -85,7 +85,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: cur = conn.cursor() csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') - print("exporting to: " + csv_name) + logger.error("experiment 10: exporting to: " + csv_name) with open(csv_name, 'w') as file: if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ @@ -93,24 +93,26 @@ def retrieve_sql_tables_as_csv(table_name, start, end): else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" cur.copy_expert(sql, file, size=8192) - logger.error("experiment 8 retrieve_tables sql: " + sql) - logger.error("experiment 8: " + sql) + logger.error("experiment 10: retrieve_tables sql: " + sql) + logger.error("experiment 10: before file close: " + sql) file.close() - logger.error("experiment 8 after file close ") + logger.error("experiment 10: after file close ") with open(csv_name, 'r') as file2: + logger.error("experiment 10: open file2 ", csv_name) csv_files[table_name] = file2.read() file2.close() - logger.error("experiment 8 after second file close ") + logger.error("experiment 10: after second file close ") os.remove(csv_name) - logger.error("experiment 8 after remove ") + logger.error("experiment 10: after remove, status ", status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 8 before conn.commit") + logger.error("experiment 10: after status +=, ", status) + logger.error("experiment 10: before conn.commit") conn.commit() - logger.error("experiment 8 after conn.commit ") + logger.error("experiment 10: after conn.commit ") conn.close() - logger.error("experiment 8 after conn.commit ") + logger.error("experiment 10: after conn.commit ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -120,14 +122,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 8 before results") + logger.error("experiment 10: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 8 results: ", results) + logger.error("experiment 10: results: ", results) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index 7e6cb96b1..eab3bfb65 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -182,6 +182,10 @@

Technical Information

{{git_commit_hash}} + + Experiment: + experiment 10 +

From d38c9e10320f48068a7c22e4afc0af2afdac1b5d Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 09:22:09 -0700 Subject: [PATCH 33/98] Experiment 10, still confirming /tmp/ dir for retrieve_sql_tables_as_csv --- retrieve_tables/controllers.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 1ac2fdd5f..7bf3d9019 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -88,8 +88,8 @@ def retrieve_sql_tables_as_csv(table_name, start, end): logger.error("experiment 10: exporting to: " + csv_name) with open(csv_name, 'w') as file: if positive_value_exists(end): - sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ - " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" cur.copy_expert(sql, file, size=8192) @@ -98,16 +98,23 @@ def retrieve_sql_tables_as_csv(table_name, start, end): file.close() logger.error("experiment 10: after file close ") with open(csv_name, 'r') as file2: - logger.error("experiment 10: open file2 ", csv_name) + logger.error("experiment 10: open file2 " + csv_name) csv_files[table_name] = file2.read() file2.close() + + # + files = os.listdir('/tmp') + files_str = '|'.join(files) + logger.error("experiment 10: /tmp dir" + files_str) + # + logger.error("experiment 10: after second file close ") os.remove(csv_name) - logger.error("experiment 10: after remove, status ", status) + logger.error("experiment 10: after remove, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 10: after status +=, ", status) + logger.error("experiment 10: after status +=, " + status) logger.error("experiment 10: before conn.commit") conn.commit() logger.error("experiment 10: after conn.commit ") @@ -216,6 +223,7 @@ def retrieve_sql_files_from_master_server(request): final_lines_count = 0 while end < 20000000: t2 = time.time() + # To test locally call https://wevotedeveloper.com:8000/apis/v1/retrieveSQLTables/?table=election_election response = requests.get("https://api.wevoteusa.org/apis/v1/retrieveSQLTables/", params={'table': table_name, 'start': start, 'end': end}) structured_json = json.loads(response.text) From dacfedf9614cb20e949440de3e1b9a2d34edd9cc Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 10:11:08 -0700 Subject: [PATCH 34/98] Experiment 11, still confirming /tmp/ dir for retrieve_sql_tables_as_csv using tempfile.mkstemp to make the temp file name/path --- retrieve_tables/controllers.py | 33 +++++++++++++++++--------------- templates/admin_tools/index.html | 2 +- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 7bf3d9019..c15a4a261 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -2,6 +2,7 @@ import json import os import re +import tempfile import psycopg2 import requests @@ -84,8 +85,10 @@ def retrieve_sql_tables_as_csv(table_name, start, end): if table_name in allowable_tables: try: cur = conn.cursor() - csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') - logger.error("experiment 10: exporting to: " + csv_name) + # csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') + fd, csv_name = tempfile.mkstemp(prefix=table_name, suffix=".csvTemp") + + logger.error("experiment 11: exporting to: " + csv_name) with open(csv_name, 'w') as file: if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ @@ -93,33 +96,33 @@ def retrieve_sql_tables_as_csv(table_name, start, end): else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" cur.copy_expert(sql, file, size=8192) - logger.error("experiment 10: retrieve_tables sql: " + sql) - logger.error("experiment 10: before file close: " + sql) + logger.error("experiment 11: retrieve_tables sql: " + sql) + logger.error("experiment 11: before file close: " + sql) file.close() - logger.error("experiment 10: after file close ") + logger.error("experiment 11: after file close ") with open(csv_name, 'r') as file2: - logger.error("experiment 10: open file2 " + csv_name) + logger.error("experiment 11: open file2 " + csv_name) csv_files[table_name] = file2.read() file2.close() # files = os.listdir('/tmp') files_str = '|'.join(files) - logger.error("experiment 10: /tmp dir" + files_str) + logger.error("experiment 11: /tmp dir" + files_str) # - logger.error("experiment 10: after second file close ") + logger.error("experiment 11: after second file close ") os.remove(csv_name) - logger.error("experiment 10: after remove, status " + status) + logger.error("experiment 11: after remove, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 10: after status +=, " + status) - logger.error("experiment 10: before conn.commit") + logger.error("experiment 11: after status +=, " + status) + logger.error("experiment 11: before conn.commit") conn.commit() - logger.error("experiment 10: after conn.commit ") + logger.error("experiment 11: after conn.commit ") conn.close() - logger.error("experiment 10: after conn.commit ") + logger.error("experiment 11: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -129,14 +132,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 10: before results") + logger.error("experiment 11: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 10: results: ", results) + logger.error("experiment 11: results: ", results) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index eab3bfb65..a6e31c5dc 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -184,7 +184,7 @@

Technical Information

Experiment: - experiment 10 + experiment 11

From bdf6702e69316bd94d214524bd49f3b21c66d592 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 10:52:35 -0700 Subject: [PATCH 35/98] Experiment 12, still confirming /tmp/ dir for retrieve_sql_tables_as_csv No longer explicitly closing file, looks like cur.copy_expert closes it after the write --- config/base.py | 2 +- retrieve_tables/controllers.py | 36 +++++++++++++++++--------------- templates/admin_tools/index.html | 2 +- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/config/base.py b/config/base.py index cac5f0aad..9b2cf4adf 100644 --- a/config/base.py +++ b/config/base.py @@ -112,7 +112,7 @@ def get_git_commit_hash(full): hash = 'git_commit_hash-file-not-found' if full: - return "https://github.com/wevote/WeVoteServer/pull/1862/commits/" + hash + return "https://github.com/wevote/WeVoteServer/commit/" + hash return hash def get_postgres_version(): diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index c15a4a261..152ecb780 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -88,7 +88,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') fd, csv_name = tempfile.mkstemp(prefix=table_name, suffix=".csvTemp") - logger.error("experiment 11: exporting to: " + csv_name) + logger.error("experiment 12: exporting to: " + csv_name) with open(csv_name, 'w') as file: if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ @@ -96,33 +96,35 @@ def retrieve_sql_tables_as_csv(table_name, start, end): else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" cur.copy_expert(sql, file, size=8192) - logger.error("experiment 11: retrieve_tables sql: " + sql) - logger.error("experiment 11: before file close: " + sql) - file.close() - logger.error("experiment 11: after file close ") + logger.error("experiment 12: retrieve_tables sql: " + sql) + logger.error("experiment 12: before file close: " + sql) + # # file.seek(0) # rewind file + # csv_files[table_name] = file.read() + # file.close() + logger.error("experiment 12: after file close (no longer explicit, looks like copy_expert closes file") with open(csv_name, 'r') as file2: - logger.error("experiment 11: open file2 " + csv_name) + logger.error("experiment 12: open file2 " + csv_name) csv_files[table_name] = file2.read() file2.close() # - files = os.listdir('/tmp') - files_str = '|'.join(files) - logger.error("experiment 11: /tmp dir" + files_str) + # files = os.listdir('/tmp') + # files_str = '|'.join(files) + # logger.error("experiment 12: /tmp dir" + files_str) # - logger.error("experiment 11: after second file close ") + logger.error("experiment 12: after second file close ") os.remove(csv_name) - logger.error("experiment 11: after remove, status " + status) + logger.error("experiment 12: after remove, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 11: after status +=, " + status) - logger.error("experiment 11: before conn.commit") + logger.error("experiment 12: after status +=, " + status) + logger.error("experiment 12: before conn.commit") conn.commit() - logger.error("experiment 11: after conn.commit ") + logger.error("experiment 12: after conn.commit ") conn.close() - logger.error("experiment 11: after conn.close ") + logger.error("experiment 12: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -132,14 +134,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 11: before results") + logger.error("experiment 12: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 11: results: ", results) + logger.error("experiment 12: results: ", results) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index a6e31c5dc..61337a380 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -184,7 +184,7 @@

Technical Information

Experiment: - experiment 11 + experiment 12

From df05c138e81916deadb7fc873643ce246e7a4c47 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 11:25:02 -0700 Subject: [PATCH 36/98] Experiment 13, eliminated temp files The way I should have done it the first time --- retrieve_tables/controllers.py | 60 ++++++++++++-------------------- templates/admin_tools/index.html | 2 +- 2 files changed, 24 insertions(+), 38 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 152ecb780..e844699af 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -3,6 +3,7 @@ import os import re import tempfile +from io import StringIO import psycopg2 import requests @@ -85,46 +86,31 @@ def retrieve_sql_tables_as_csv(table_name, start, end): if table_name in allowable_tables: try: cur = conn.cursor() - # csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') - fd, csv_name = tempfile.mkstemp(prefix=table_name, suffix=".csvTemp") - - logger.error("experiment 12: exporting to: " + csv_name) - with open(csv_name, 'w') as file: - if positive_value_exists(end): - sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ - end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - else: - sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 12: retrieve_tables sql: " + sql) - logger.error("experiment 12: before file close: " + sql) - # # file.seek(0) # rewind file - # csv_files[table_name] = file.read() - # file.close() - logger.error("experiment 12: after file close (no longer explicit, looks like copy_expert closes file") - with open(csv_name, 'r') as file2: - logger.error("experiment 12: open file2 " + csv_name) - csv_files[table_name] = file2.read() - file2.close() - - # - # files = os.listdir('/tmp') - # files_str = '|'.join(files) - # logger.error("experiment 12: /tmp dir" + files_str) - # - - logger.error("experiment 12: after second file close ") - os.remove(csv_name) - logger.error("experiment 12: after remove, status " + status) + file = StringIO() # Empty file + + logger.error("experiment 13: file.closed: " + str(file)) + if positive_value_exists(end): + sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + else: + sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 13: retrieve_tables sql: " + sql) + # file.seek(0) + # logger.error("experiment 13: retrieve_tables file contents: " + file.read()) + file.seek(0) + csv_files[table_name] = file.read() + file.close() + logger.error("experiment 13: after file close, status "+ status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 12: after status +=, " + status) - logger.error("experiment 12: before conn.commit") + logger.error("experiment 13: after status +=, " + status) + logger.error("experiment 13: before conn.commit") conn.commit() - logger.error("experiment 12: after conn.commit ") + logger.error("experiment 13: after conn.commit ") conn.close() - logger.error("experiment 12: after conn.close ") + logger.error("experiment 13: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -134,14 +120,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 12: before results") + logger.error("experiment 13: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 12: results: ", results) + logger.error("experiment 13: results: " + str(results)) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index 61337a380..a08fc708c 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -184,7 +184,7 @@

Technical Information

Experiment: - experiment 12 + experiment 13

From c7873574f8e728be38cdcf9ba4117b2b1b401f6d Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 12:54:00 -0700 Subject: [PATCH 37/98] Experiment 14, narrowing in on psycopg2-binary issue --- retrieve_tables/controllers.py | 45 ++++++++++++++++++++++---------- templates/admin_tools/index.html | 2 +- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index e844699af..71a327b59 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -2,15 +2,15 @@ import json import os import re -import tempfile +import time from io import StringIO import psycopg2 import requests -import time -from config.base import get_environment_variable from django.http import HttpResponse + import wevote_functions.admin +from config.base import get_environment_variable from wevote_functions.functions import positive_value_exists logger = wevote_functions.admin.get_logger(__name__) @@ -71,6 +71,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = '' + f = open("requirements.txt", "r") + for line in f: + if "psycopg2" in line: + logger.error("experiment 14: psycopg2: " + line.strip()) + try: conn = psycopg2.connect( database=get_environment_variable('DATABASE_NAME'), @@ -82,35 +87,47 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # logger.debug("retrieve_sql_tables_as_csv psycopg2 Connected to DB") + try: + # Simple copy experiment + sql = 'COPY "election_election" TO STDOUT;' + file = StringIO() # Empty file + cur = conn.cursor() + cur.copy_expert(sql, file, size=8192) + file.seek(0) + logger.error("experiment 14: select some stuff: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + csv_files = {} if table_name in allowable_tables: try: cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 13: file.closed: " + str(file)) + logger.error("experiment 14: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + logger.error("experiment 14: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 13: retrieve_tables sql: " + sql) - # file.seek(0) - # logger.error("experiment 13: retrieve_tables file contents: " + file.read()) + logger.error("experiment 14: after cur.copy_expert ") + file.seek(0) + logger.error("experiment 14: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 13: after file close, status "+ status) + logger.error("experiment 14: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 13: after status +=, " + status) - logger.error("experiment 13: before conn.commit") + logger.error("experiment 14: after status +=, " + status) + logger.error("experiment 14: before conn.commit") conn.commit() - logger.error("experiment 13: after conn.commit ") + logger.error("experiment 14: after conn.commit ") conn.close() - logger.error("experiment 13: after conn.close ") + logger.error("experiment 14: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -120,14 +137,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 13: before results") + logger.error("experiment 14: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 13: results: " + str(results)) + logger.error("experiment 14: results: " + str(results)) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index a08fc708c..0e151190e 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -184,7 +184,7 @@

Technical Information

Experiment: - experiment 13 + experiment 14

From 4395eeb5dc15ce049cd02908bd9d1c5e52eacdf4 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 15:02:56 -0700 Subject: [PATCH 38/98] Experiment 15, exploring copy_expert simplification --- retrieve_tables/controllers.py | 41 +++++++---- templates/admin_tools/index.html | 4 -- voter_guide/controllers.py | 116 +++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+), 17 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 71a327b59..4e9482401 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 14: psycopg2: " + line.strip()) + logger.error("experiment 15: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -94,7 +94,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 14: select some stuff: " + file.readline().strip()) + logger.error("experiment 15: select some stuff: " + file.readline().strip()) except Exception as e: logger.error("Real exception in select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") @@ -104,30 +104,45 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 14: file: " + str(file)) + if positive_value_exists(end): + sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + end + " ORDER BY id) TO STDOUT'" + else: + sql = "COPY " + table_name + " TO STDOUT" + logger.error("experiment 15: SIMPLIFIED retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 15: SIMPLIFIED after cur.copy_expert ") + file.seek(0) + logger.error("experiment 15: SIMPLIFIED retrieve_tables file contents: " + file.readline().strip()) + + + cur = conn.cursor() + file = StringIO() # Empty file + + logger.error("experiment 15: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 14: retrieve_tables sql: " + sql) + logger.error("experiment 15: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 14: after cur.copy_expert ") + logger.error("experiment 15: after cur.copy_expert ") file.seek(0) - logger.error("experiment 14: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 15: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 14: after file close, status " + status) + logger.error("experiment 15: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 14: after status +=, " + status) - logger.error("experiment 14: before conn.commit") + logger.error("experiment 15: after status +=, " + status) + logger.error("experiment 15: before conn.commit") conn.commit() - logger.error("experiment 14: after conn.commit ") + logger.error("experiment 15: after conn.commit ") conn.close() - logger.error("experiment 14: after conn.close ") + logger.error("experiment 15: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -137,14 +152,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 14: before results") + logger.error("experiment 15: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 14: results: " + str(results)) + logger.error("experiment 15: results: " + str(results)) return results except Exception as e: diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index 0e151190e..7e6cb96b1 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -182,10 +182,6 @@

Technical Information

{{git_commit_hash}} - - Experiment: - experiment 14 -

diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 7a8247673..5b080fda3 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -2112,6 +2112,10 @@ def voter_guide_possibility_positions_retrieve_for_api( # voterGuidePossibility results = voter_guide_possibility_manager.retrieve_voter_guide_possibility( voter_guide_possibility_id=voter_guide_possibility_id) + # TODO: Steve, move all the voterGuidPossiblityPositions with the same organization_we_vote_id that was changed in the last 6 months + # move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(voter_guide_possibility_id) + + possible_endorsement_list = [] if results['voter_guide_possibility_found']: voter_guide_possibility = results['voter_guide_possibility'] @@ -2206,6 +2210,118 @@ def voter_guide_possibility_positions_retrieve_for_api( # voterGuidePossibility return json_data + # def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(voter_guide_possibility_id): + # voter_guide_possibility_query = VoterGuidePossibility.objects.filter( + # Q(voter_guide_possibility_url__iexact=voter_guide_possibility_url) | + # Q(voter_guide_possibility_url__iexact=voter_guide_possibility_url_alternate)) + # # DALE 2020-06-08 After working with this, it is better to include entries hidden from active review + # # voter_guide_possibility_query = voter_guide_possibility_query.exclude(hide_from_active_review=True) + # + # # Only retrieve by URL if it was created this year + # now = datetime.now() + # status += "LIMITING_TO_THIS_YEAR: " + str(now.year) + " " + # voter_guide_possibility_query = voter_guide_possibility_query.filter(date_last_changed__year=now.year) + # + # voter_guide_possibility_on_stage = voter_guide_possibility_query.last() + # if voter_guide_possibility_on_stage is not None: + # voter_guide_possibility_on_stage_id = voter_guide_possibility_on_stage.id + # status += "VOTER_GUIDE_POSSIBILITY_FOUND_WITH_URL " + # success = True + # else: + # status += "VOTER_GUIDE_POSSIBILITY_NOT_FOUND_WITH_URL " + # success = True + # + # possibility_position_query = VoterGuidePossibilityPosition.objects.filter( + # voter_guide_possibility_parent_id=voter_guide_possibility.id).order_by('possibility_position_number') + # possibility_position_list = list(possibility_position_query) + # for possibility_position in possibility_position_list: + # if positive_value_exists(possibility_position.more_info_url): + # more_info_url = possibility_position.more_info_url + # else: + # more_info_url = voter_guide_possibility.voter_guide_possibility_url + # if voter_guide_possibility.voter_guide_possibility_type == ORGANIZATION_ENDORSING_CANDIDATES \ + # or voter_guide_possibility.voter_guide_possibility_type == UNKNOWN_TYPE: + # # The organization variables have been set above + # # Note that UNKNOWN_TYPE might be set if we are looking at organization + # # ###################### + # # If we are starting from a single organization endorsing many candidates, + # # we store that candidate information once + # if positive_value_exists(possibility_position.candidate_we_vote_id): + # candidate_name = possibility_position.ballot_item_name + # position_json = { + # 'candidate_name': candidate_name, + # 'candidate_we_vote_id': possibility_position.candidate_we_vote_id, + # 'candidate_twitter_handle': possibility_position.candidate_twitter_handle, + # 'google_civic_election_id': possibility_position.google_civic_election_id, + # 'more_info_url': more_info_url, + # 'organization_name': organization_name, + # 'organization_we_vote_id': organization_we_vote_id, + # 'organization_twitter_handle': organization_twitter_handle, + # 'stance': possibility_position.position_stance, + # 'statement_text': possibility_position.statement_text, + # 'state_code': state_code, + # } + # position_json_list.append(position_json) + # elif positive_value_exists(possibility_position.measure_we_vote_id): + # measure_title = possibility_position.ballot_item_name + # position_json = { + # 'google_civic_election_id': possibility_position.google_civic_election_id, + # 'measure_title': measure_title, + # 'measure_we_vote_id': possibility_position.measure_we_vote_id, + # 'more_info_url': more_info_url, + # 'organization_name': organization_name, + # 'organization_we_vote_id': organization_we_vote_id, + # 'organization_twitter_handle': organization_twitter_handle, + # 'stance': possibility_position.position_stance, + # 'statement_text': possibility_position.statement_text, + # 'state_code': state_code, + # } + # position_json_list.append(position_json) + # else: + # status += "MISSING_BOTH_CANDIDATE_AND_MEASURE_WE_VOTE_ID " + # elif voter_guide_possibility.voter_guide_possibility_type == ENDORSEMENTS_FOR_CANDIDATE: + # # ###################### + # # If we are starting from a single candidate endorsed by many "organizations" (which may be people), + # # we store the unique organization information in the VoterGuidePossibilityPosition table + # organization_name = possibility_position.organization_name \ + # if positive_value_exists(possibility_position.organization_name) else "" + # organization_we_vote_id = possibility_position.organization_we_vote_id \ + # if positive_value_exists(possibility_position.organization_we_vote_id) else "" + # organization_twitter_handle = possibility_position.organization_twitter_handle \ + # if positive_value_exists(possibility_position.organization_twitter_handle) else "" + # position_json = { + # 'candidate_name': candidate_name, + # 'candidate_we_vote_id': candidate_we_vote_id, + # 'candidate_twitter_handle': candidate_twitter_handle, + # 'contest_office_name': contest_office_name, + # 'google_civic_election_id': possibility_position.google_civic_election_id, + # 'measure_we_vote_id': possibility_position.measure_we_vote_id, + # 'more_info_url': more_info_url, + # 'organization_name': organization_name, + # 'organization_we_vote_id': organization_we_vote_id, + # 'organization_twitter_handle': organization_twitter_handle, + # 'stance': possibility_position.position_stance, + # 'statement_text': possibility_position.statement_text, + # 'state_code': state_code, + # } + # position_json_list.append(position_json) + # else: + # # This is an error condition which should not be reached + # status += "UNRECOGNIZED_VOTER_POSSIBILITY_TYPE " + # + # if len(position_json_list): + # position_json_list_found = True + # + # results = { + # 'status': status, + # 'success': success, + # 'position_json_list': position_json_list, + # 'position_json_list_found': position_json_list_found, + # } + # return results + + + def voter_guide_possibility_save_for_api( # voterGuidePossibilitySave voter_device_id, voter_guide_possibility_id, From e483f73529ba07367f372a20fb58c8692dde1eee Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Wed, 13 Jul 2022 16:04:09 -0700 Subject: [PATCH 39/98] Changed election list sort to pay attention to election name order, so "A" states show up first and "W" states show up later. --- apis_v1/views/views_voter.py | 1 + election/models.py | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index 3456a9522..fa7c3029f 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -3039,6 +3039,7 @@ def voter_verify_secret_code_view(request): # voterVerifySecretCode } return HttpResponse(json.dumps(json_data), content_type='application/json') + @csrf_exempt def voter_contact_list_retrieve_view(request): # voterContactListRetrieve """ diff --git a/election/models.py b/election/models.py index 255a4a718..6e1471a81 100644 --- a/election/models.py +++ b/election/models.py @@ -110,7 +110,7 @@ class Election(models.Model): ignore_this_election = models.BooleanField(default=False) # Have we finished all the election preparation related to Offices, Candidates, Measures and Ballot Locations? election_preparation_finished = models.BooleanField(default=False) - # Have we finished all of the election preparation related to Candidate photos? + # Have we finished all the election preparation related to Candidate photos? candidate_photos_finished = models.BooleanField(default=False) # This is a multi-state election @@ -433,7 +433,7 @@ def retrieve_elections_by_date(self, newest_to_oldest=True, include_test_electio election_list_query = Election.objects.using('readonly').all() if not positive_value_exists(include_test_election): election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_name', 'election_day_text') if newest_to_oldest: # Typically we want the newest election displayed first, with the older elections later in the list election_list_query = election_list_query.reverse() @@ -528,7 +528,7 @@ def retrieve_election( def retrieve_listed_elections(self): """ - These are all of the elections marked as "listed" with "include_in_list_for_voters" + These are all the elections marked as "listed" with "include_in_list_for_voters" :return: """ election_list = [] @@ -575,7 +575,7 @@ def retrieve_upcoming_elections( election_list_query = election_list_query.filter(state_code__iexact=state_code) if not positive_value_exists(include_test_election): election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_name', 'election_day_text') upcoming_election_list = list(election_list_query) @@ -655,7 +655,7 @@ def retrieve_prior_elections_this_year(self, state_code="", without_state_code=F elif positive_value_exists(state_code): election_list_query = election_list_query.filter(state_code__iexact=state_code) election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_name', 'election_day_text') prior_election_list = list(election_list_query) @@ -1105,7 +1105,7 @@ def retrieve_elections_by_election_date(self, election_day_text='', include_test election_list_query = election_list_query.exclude(google_civic_election_id=2000) if election_day_text: election_list_query = election_list_query.filter(election_day_text=election_day_text) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_name', 'election_day_text') election_list = election_list_query status = 'ELECTIONS_FOUND' success = True @@ -1185,7 +1185,7 @@ def retrieve_elections_by_state_and_election_date(self, state_code='', election_ if state_code and election_day_text: election_list_query = election_list_query.filter(state_code__iexact=state_code, election_day_text=election_day_text) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_name', 'election_day_text') election_list = election_list_query status = 'ELECTIONS_FOUND' success = True From 043b5e73f7e56cd219ad54365b0f8130e9863c48 Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Wed, 13 Jul 2022 17:24:15 -0700 Subject: [PATCH 40/98] Changed election list sort to pay attention to election name order, so "A" states show up first and "W" states show up later. Date, oldest to newest is main sort, with election_name as secondary sort. --- election/models.py | 10 +++++----- election/views_admin.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/election/models.py b/election/models.py index 6e1471a81..271384f18 100644 --- a/election/models.py +++ b/election/models.py @@ -433,7 +433,7 @@ def retrieve_elections_by_date(self, newest_to_oldest=True, include_test_electio election_list_query = Election.objects.using('readonly').all() if not positive_value_exists(include_test_election): election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_name', 'election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') if newest_to_oldest: # Typically we want the newest election displayed first, with the older elections later in the list election_list_query = election_list_query.reverse() @@ -575,7 +575,7 @@ def retrieve_upcoming_elections( election_list_query = election_list_query.filter(state_code__iexact=state_code) if not positive_value_exists(include_test_election): election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_name', 'election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') upcoming_election_list = list(election_list_query) @@ -655,7 +655,7 @@ def retrieve_prior_elections_this_year(self, state_code="", without_state_code=F elif positive_value_exists(state_code): election_list_query = election_list_query.filter(state_code__iexact=state_code) election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_name', 'election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') prior_election_list = list(election_list_query) @@ -1105,7 +1105,7 @@ def retrieve_elections_by_election_date(self, election_day_text='', include_test election_list_query = election_list_query.exclude(google_civic_election_id=2000) if election_day_text: election_list_query = election_list_query.filter(election_day_text=election_day_text) - election_list_query = election_list_query.order_by('election_name', 'election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') election_list = election_list_query status = 'ELECTIONS_FOUND' success = True @@ -1185,7 +1185,7 @@ def retrieve_elections_by_state_and_election_date(self, state_code='', election_ if state_code and election_day_text: election_list_query = election_list_query.filter(state_code__iexact=state_code, election_day_text=election_day_text) - election_list_query = election_list_query.order_by('election_name', 'election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') election_list = election_list_query status = 'ELECTIONS_FOUND' success = True diff --git a/election/views_admin.py b/election/views_admin.py index ef0669c53..9d289760b 100644 --- a/election/views_admin.py +++ b/election/views_admin.py @@ -864,7 +864,7 @@ def election_list_view(request): office_manager = ContestOfficeManager() election_list_query = Election.objects.all() - election_list_query = election_list_query.order_by('election_name', 'election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') election_list_query = election_list_query.exclude(google_civic_election_id=2000) if positive_value_exists(show_ignored_elections): # Do not filter out ignored elections From a14bb82d707be24eca43b4e685c860cc95e12644 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 20:04:30 -0700 Subject: [PATCH 41/98] Experiment 16, exploring copy_expert simplification, part 2 --- retrieve_tables/controllers.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 4e9482401..7f0b179f5 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 15: psycopg2: " + line.strip()) + logger.error("experiment 16: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -94,7 +94,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 15: select some stuff: " + file.readline().strip()) + logger.error("experiment 16: select some stuff: " + file.readline().strip()) except Exception as e: logger.error("Real exception in select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") @@ -109,40 +109,40 @@ def retrieve_sql_tables_as_csv(table_name, start, end): end + " ORDER BY id) TO STDOUT'" else: sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 15: SIMPLIFIED retrieve_tables sql: " + sql) + logger.error("experiment 16: SIMPLIFIED retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 15: SIMPLIFIED after cur.copy_expert ") + logger.error("experiment 16: SIMPLIFIED after cur.copy_expert ") file.seek(0) - logger.error("experiment 15: SIMPLIFIED retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 16: SIMPLIFIED retrieve_tables file contents: " + file.readline().strip()) cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 15: file: " + str(file)) + logger.error("experiment 16: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 15: retrieve_tables sql: " + sql) + logger.error("experiment 16: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 15: after cur.copy_expert ") + logger.error("experiment 16: after cur.copy_expert ") file.seek(0) - logger.error("experiment 15: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 16: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 15: after file close, status " + status) + logger.error("experiment 16: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 15: after status +=, " + status) - logger.error("experiment 15: before conn.commit") + logger.error("experiment 16: after status +=, " + status) + logger.error("experiment 16: before conn.commit") conn.commit() - logger.error("experiment 15: after conn.commit ") + logger.error("experiment 16: after conn.commit ") conn.close() - logger.error("experiment 15: after conn.close ") + logger.error("experiment 16: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -152,14 +152,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 15: before results") + logger.error("experiment 16: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 15: results: " + str(results)) + logger.error("experiment 16: results: " + str(results)) return results except Exception as e: From 30beefc9f990cdf6a10865b1ab98d7a2b6efb2c5 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Wed, 13 Jul 2022 20:07:09 -0700 Subject: [PATCH 42/98] Experiment 16, exploring copy_expert simplification, part 2 --- retrieve_tables/controllers.py | 2 +- voter_guide/controllers.py | 31 ++++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 7f0b179f5..e0ccb12c7 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -106,7 +106,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ - end + " ORDER BY id) TO STDOUT'" + end + " ORDER BY id) TO STDOUT" else: sql = "COPY " + table_name + " TO STDOUT" logger.error("experiment 16: SIMPLIFIED retrieve_tables sql: " + sql) diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 5b080fda3..86b520927 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -1,6 +1,7 @@ # voter_guide/controllers.py # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- +from urllib.parse import urlparse from ballot.models import OFFICE, CANDIDATE, MEASURE from candidate.controllers import retrieve_candidate_list_for_all_prior_elections_this_year, \ @@ -8,7 +9,7 @@ from candidate.models import CandidateManager, CandidateListManager from config.base import get_environment_variable import copy -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date from django.http import HttpResponse from election.controllers import retrieve_this_years_election_id_list, retrieve_upcoming_election_id_list from election.models import ElectionManager @@ -2113,7 +2114,7 @@ def voter_guide_possibility_positions_retrieve_for_api( # voterGuidePossibility voter_guide_possibility_id=voter_guide_possibility_id) # TODO: Steve, move all the voterGuidPossiblityPositions with the same organization_we_vote_id that was changed in the last 6 months - # move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(voter_guide_possibility_id) + move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(results['voter_guide_possibility']) possible_endorsement_list = [] @@ -2210,17 +2211,29 @@ def voter_guide_possibility_positions_retrieve_for_api( # voterGuidePossibility return json_data - # def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(voter_guide_possibility_id): - # voter_guide_possibility_query = VoterGuidePossibility.objects.filter( - # Q(voter_guide_possibility_url__iexact=voter_guide_possibility_url) | - # Q(voter_guide_possibility_url__iexact=voter_guide_possibility_url_alternate)) - # # DALE 2020-06-08 After working with this, it is better to include entries hidden from active review +def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(voter_guide_possibility): + voter_guide_possibility_id = voter_guide_possibility.id + organization_we_vote_id = voter_guide_possibility.organization_we_vote_id + voter_guide_possibility_url = voter_guide_possibility.voter_guide_possibility_url + url_root = voter_guide_possibility_url.split('/')[2] + + enddate = date.today() + startdate = enddate - timedelta(months=6) + + voter_guide_possibility_query = VoterGuidePossibility.objects.filter( + Q(voter_guide_possibility_url__contains=url_root) & + Q(organization_we_vote_id__iexact=organization_we_vote_id) & + Q(date_last_changed__range=[startdate, enddate])).exclude(id=voter_guide_possibility_id) + + # now itterate through this list, and move all the lastest_voter_guide_possibility ies to this voter_guide_possibility_id as parent + + # voter_guide_possibility_query = voter_guide_possibility_query.filter(date_last_changed__year=now.year) + # # DALE 2020-06-08 After working with this, it is better to include entries hidden from active review # # voter_guide_possibility_query = voter_guide_possibility_query.exclude(hide_from_active_review=True) # # # Only retrieve by URL if it was created this year - # now = datetime.now() + # own = datetime.now() # status += "LIMITING_TO_THIS_YEAR: " + str(now.year) + " " - # voter_guide_possibility_query = voter_guide_possibility_query.filter(date_last_changed__year=now.year) # # voter_guide_possibility_on_stage = voter_guide_possibility_query.last() # if voter_guide_possibility_on_stage is not None: From f269f45a2ce2accc1cd046ac43c9d9e5f73ec702 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 09:33:19 -0700 Subject: [PATCH 43/98] Experiment 17, exploring copy_expert simplification, part 3 --- retrieve_tables/controllers.py | 71 ++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 17 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index e0ccb12c7..974876116 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 16: psycopg2: " + line.strip()) + logger.error("experiment 17: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -94,7 +94,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 16: select some stuff: " + file.readline().strip()) + logger.error("experiment 17: select some stuff: " + file.readline().strip()) except Exception as e: logger.error("Real exception in select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") @@ -103,46 +103,83 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: cur = conn.cursor() file = StringIO() # Empty file + if positive_value_exists(end): + sql = "COPY public.candidate_candidatecampaign TO STDOUT" + else: + sql = "COPY " + table_name + " TO STDOUT" + logger.error("experiment 17: SIMPLIFIED4 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 17: SIMPLIFIED4 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 17: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + cur = conn.cursor() + file = StringIO() # Empty file if positive_value_exists(end): - sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" + else: + sql = "COPY " + table_name + " TO STDOUT" + logger.error("experiment 17: SIMPLIFIED3 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 17: SIMPLIFIED3 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 17: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + + cur = conn.cursor() + file = StringIO() # Empty file + if positive_value_exists(end): + sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" + else: + sql = "COPY " + table_name + " TO STDOUT" + logger.error("experiment 17: SIMPLIFIED2 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 17: SIMPLIFIED2 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 17: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + + cur = conn.cursor() + file = StringIO() # Empty file + if positive_value_exists(end): + sql = "COPY (SELECT * FROM \"public\".\"" + table_name + "\" WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT" + # sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + # end + " ORDER BY id) TO STDOUT" else: sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 16: SIMPLIFIED retrieve_tables sql: " + sql) + logger.error("experiment 17: SIMPLIFIED retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 16: SIMPLIFIED after cur.copy_expert ") + logger.error("experiment 17: SIMPLIFIED after cur.copy_expert ") file.seek(0) - logger.error("experiment 16: SIMPLIFIED retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 17: SIMPLIFIED retrieve_tables file contents: " + file.readline().strip()) cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 16: file: " + str(file)) + logger.error("experiment 17: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 16: retrieve_tables sql: " + sql) + logger.error("experiment 17: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 16: after cur.copy_expert ") + logger.error("experiment 17: after cur.copy_expert ") file.seek(0) - logger.error("experiment 16: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 17: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 16: after file close, status " + status) + logger.error("experiment 17: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 16: after status +=, " + status) - logger.error("experiment 16: before conn.commit") + logger.error("experiment 17: after status +=, " + status) + logger.error("experiment 17: before conn.commit") conn.commit() - logger.error("experiment 16: after conn.commit ") + logger.error("experiment 17: after conn.commit ") conn.close() - logger.error("experiment 16: after conn.close ") + logger.error("experiment 17: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -152,14 +189,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 16: before results") + logger.error("experiment 17: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 16: results: " + str(results)) + logger.error("experiment 17: results: " + str(results)) return results except Exception as e: From 8db66a637cd17ab25695e31a8db2935ad81f4058 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 09:35:08 -0700 Subject: [PATCH 44/98] Experiment 17, exploring copy_expert simplification, part 3 --- voter_guide/controllers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 86b520927..f3e35c1e1 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -3,6 +3,8 @@ # -*- coding: UTF-8 -*- from urllib.parse import urlparse +from django.db.models import Q + from ballot.models import OFFICE, CANDIDATE, MEASURE from candidate.controllers import retrieve_candidate_list_for_all_prior_elections_this_year, \ retrieve_candidate_list_for_all_upcoming_elections From dda51927d802ea566c7303c5a7953b8f070e3759 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 10:20:13 -0700 Subject: [PATCH 45/98] Experiment 18, exploring copy_expert simplification --- retrieve_tables/controllers.py | 169 ++++++++++++++++++++------------- voter_guide/controllers.py | 4 + 2 files changed, 109 insertions(+), 64 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 974876116..aeccc7388 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 17: psycopg2: " + line.strip()) + logger.error("experiment 18: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -90,96 +90,137 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: # Simple copy experiment sql = 'COPY "election_election" TO STDOUT;' + logger.error("experiment 18: SIMPLIFIED8 sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 17: select some stuff: " + file.readline().strip()) + logger.error("experiment 18: SIMPLIFIED8 select some stuff: " + file.readline().strip()) except Exception as e: - logger.error("Real exception in select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + logger.error("Real exception in SIMPLIFIED8 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") - csv_files = {} - if table_name in allowable_tables: - try: - cur = conn.cursor() - file = StringIO() # Empty file - if positive_value_exists(end): - sql = "COPY public.candidate_candidatecampaign TO STDOUT" - else: - sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 17: SIMPLIFIED4 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 17: SIMPLIFIED4 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 17: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + # Works to here - cur = conn.cursor() - file = StringIO() # Empty file - if positive_value_exists(end): - sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - else: - sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 17: SIMPLIFIED3 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 17: SIMPLIFIED3 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 17: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" + logger.error("experiment 18: SIMPLIFIED7 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 18: SIMPLIFIED7 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 18: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED7: " + str(e)) - cur = conn.cursor() - file = StringIO() # Empty file - if positive_value_exists(end): - sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - else: - sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 17: SIMPLIFIED2 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 17: SIMPLIFIED2 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 17: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + try: + # Simple copy experiment + sql = 'COPY "public"."election_election" TO STDOUT;' + logger.error("experiment 18: SIMPLIFIED6 retrieve_tables sql: " + sql) + file = StringIO() # Empty file + cur = conn.cursor() + cur.copy_expert(sql, file, size=8192) + file.seek(0) + logger.error("experiment 18: SIMPLIFIED6: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") - cur = conn.cursor() - file = StringIO() # Empty file - if positive_value_exists(end): - sql = "COPY (SELECT * FROM \"public\".\"" + table_name + "\" WHERE id BETWEEN " + start + " AND " + \ - end + " ORDER BY id) TO STDOUT" - # sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ - # end + " ORDER BY id) TO STDOUT" - else: - sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 17: SIMPLIFIED retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 17: SIMPLIFIED after cur.copy_expert ") - file.seek(0) - logger.error("experiment 17: SIMPLIFIED retrieve_tables file contents: " + file.readline().strip()) + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" + logger.error("experiment 18: SIMPLIFIED5 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 18: SIMPLIFIED5 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 18: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED5: " + str(e)) + + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" + logger.error("experiment 18: SIMPLIFIED4 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 18: SIMPLIFIED4 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 18: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED4: " + str(e)) + + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = "COPY public.candidate_candidatecampaign TO STDOUT" + logger.error("experiment 18: SIMPLIFIED3 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 18: SIMPLIFIED3 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 18: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED3: " + str(e)) + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" + logger.error("experiment 18: SIMPLIFIED2 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 18: SIMPLIFIED2 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 18: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED2: " + str(e)) + try: + cur = conn.cursor() + file = StringIO() # Empty file + if positive_value_exists(end): + sql = "COPY (SELECT * FROM \"public\".\"" + table_name + "\" WHERE id BETWEEN " + start + " AND " + \ + end + " ORDER BY id) TO STDOUT" + # sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + # end + " ORDER BY id) TO STDOUT" + else: + sql = "COPY " + table_name + " TO STDOUT" + logger.error("experiment 18: SIMPLIFIED1 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 18: SIMPLIFIED1 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 18: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED: " + str(e)) + + csv_files = {} + if table_name in allowable_tables: + try: cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 17: file: " + str(file)) + logger.error("experiment 18: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 17: retrieve_tables sql: " + sql) + logger.error("experiment 18: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 17: after cur.copy_expert ") + logger.error("experiment 18: after cur.copy_expert ") file.seek(0) - logger.error("experiment 17: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 18: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 17: after file close, status " + status) + logger.error("experiment 18: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 17: after status +=, " + status) - logger.error("experiment 17: before conn.commit") + logger.error("experiment 18: after status +=, " + status) + logger.error("experiment 18: before conn.commit") conn.commit() - logger.error("experiment 17: after conn.commit ") + logger.error("experiment 18: after conn.commit ") conn.close() - logger.error("experiment 17: after conn.close ") + logger.error("experiment 18: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -189,14 +230,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 17: before results") + logger.error("experiment 18: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 17: results: " + str(results)) + logger.error("experiment 18: results: " + str(results)) return results except Exception as e: diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index f3e35c1e1..0595ab90b 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -2226,6 +2226,10 @@ def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(vo Q(voter_guide_possibility_url__contains=url_root) & Q(organization_we_vote_id__iexact=organization_we_vote_id) & Q(date_last_changed__range=[startdate, enddate])).exclude(id=voter_guide_possibility_id) + voter_guide_possibility_list = list(voter_guide_possibility_query) + for voter_guide_possibility in voter_guide_possibility_list: + # move all the latest voter_guide_possibility ies to this voter_guide_possibility_id as parent + print(voter_guide_possibility) # now itterate through this list, and move all the lastest_voter_guide_possibility ies to this voter_guide_possibility_id as parent From d2f2f46c40897ed9eeb3d5c57206ef89d65b6861 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 11:25:23 -0700 Subject: [PATCH 46/98] Experiment 19, exploring copy_expert simplification --- friend/models.py | 1 + retrieve_tables/controllers.py | 82 +++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/friend/models.py b/friend/models.py index 28a1312eb..17636ebf9 100644 --- a/friend/models.py +++ b/friend/models.py @@ -1799,6 +1799,7 @@ def fetch_voters_with_friends_dataset_improved(self): port=get_environment_variable('DATABASE_PORT') ) cur = conn.cursor() + # July 14, 2022: If this works locally but not in AWS, then consider dropping the "public". part sql_viewer = \ 'SELECT id, COUNT(*) AS we_count FROM ( ' \ 'SELECT viewee_voter_we_vote_id id FROM "public"."friend_currentfriend" ' \ diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index aeccc7388..cc954184c 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 18: psycopg2: " + line.strip()) + logger.error("experiment 19: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -90,38 +90,50 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: # Simple copy experiment sql = 'COPY "election_election" TO STDOUT;' - logger.error("experiment 18: SIMPLIFIED8 sql: " + sql) + logger.error("experiment 19: SIMPLIFIED9 sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 18: SIMPLIFIED8 select some stuff: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED9 select some stuff: " + file.readline().strip()) except Exception as e: - logger.error("Real exception in SIMPLIFIED8 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + logger.error("Real exception in SIMPLIFIED9 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") # Works to here + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = 'COPY "party_party" TO STDOUT' + logger.error("experiment 19: SIMPLIFIED8 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 19: SIMPLIFIED8 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 19: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED7: " + str(e)) + try: cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 18: SIMPLIFIED7 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED7 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: SIMPLIFIED7 after cur.copy_expert ") + logger.error("experiment 19: SIMPLIFIED7 after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED7: " + str(e)) try: # Simple copy experiment sql = 'COPY "public"."election_election" TO STDOUT;' - logger.error("experiment 18: SIMPLIFIED6 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED6 retrieve_tables sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 18: SIMPLIFIED6: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED6: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") @@ -129,11 +141,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 18: SIMPLIFIED5 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED5 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: SIMPLIFIED5 after cur.copy_expert ") + logger.error("experiment 19: SIMPLIFIED5 after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED5: " + str(e)) @@ -141,11 +153,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 18: SIMPLIFIED4 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED4 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: SIMPLIFIED4 after cur.copy_expert ") + logger.error("experiment 19: SIMPLIFIED4 after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED4: " + str(e)) @@ -153,11 +165,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY public.candidate_candidatecampaign TO STDOUT" - logger.error("experiment 18: SIMPLIFIED3 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED3 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: SIMPLIFIED3 after cur.copy_expert ") + logger.error("experiment 19: SIMPLIFIED3 after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED3: " + str(e)) @@ -165,11 +177,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - logger.error("experiment 18: SIMPLIFIED2 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED2 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: SIMPLIFIED2 after cur.copy_expert ") + logger.error("experiment 19: SIMPLIFIED2 after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED2: " + str(e)) @@ -183,11 +195,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # end + " ORDER BY id) TO STDOUT" else: sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 18: SIMPLIFIED1 retrieve_tables sql: " + sql) + logger.error("experiment 19: SIMPLIFIED1 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: SIMPLIFIED1 after cur.copy_expert ") + logger.error("experiment 19: SIMPLIFIED1 after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED: " + str(e)) @@ -197,30 +209,30 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 18: file: " + str(file)) + logger.error("experiment 19: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 18: retrieve_tables sql: " + sql) + logger.error("experiment 19: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 18: after cur.copy_expert ") + logger.error("experiment 19: after cur.copy_expert ") file.seek(0) - logger.error("experiment 18: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 19: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 18: after file close, status " + status) + logger.error("experiment 19: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 18: after status +=, " + status) - logger.error("experiment 18: before conn.commit") + logger.error("experiment 19: after status +=, " + status) + logger.error("experiment 19: before conn.commit") conn.commit() - logger.error("experiment 18: after conn.commit ") + logger.error("experiment 19: after conn.commit ") conn.close() - logger.error("experiment 18: after conn.close ") + logger.error("experiment 19: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -230,14 +242,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 18: before results") + logger.error("experiment 19: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 18: results: " + str(results)) + logger.error("experiment 19: results: " + str(results)) return results except Exception as e: From e521a5eea8f1cfc0c3882cece13ade98f0da1ce8 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 12:12:03 -0700 Subject: [PATCH 47/98] Experiment 20, exploring copy_expert simplification --- retrieve_tables/controllers.py | 110 +++++++++++++++++++++------------ 1 file changed, 70 insertions(+), 40 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index cc954184c..c5742f462 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 19: psycopg2: " + line.strip()) + logger.error("experiment 20: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -90,50 +90,80 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: # Simple copy experiment sql = 'COPY "election_election" TO STDOUT;' - logger.error("experiment 19: SIMPLIFIED9 sql: " + sql) + logger.error("experiment 20: SIMPLIFIED11 sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 19: SIMPLIFIED9 select some stuff: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED11 select some stuff: " + file.readline().strip()) except Exception as e: - logger.error("Real exception in SIMPLIFIED9 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + logger.error("Real exception in SIMPLIFIED11 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = 'COPY "party_party" TO STDOUT' + logger.error("experiment 20: SIMPLIFIED10 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 20: SIMPLIFIED10 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 20: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED10: " + str(e)) # Works to here try: cur = conn.cursor() file = StringIO() # Empty file - sql = 'COPY "party_party" TO STDOUT' - logger.error("experiment 19: SIMPLIFIED8 retrieve_tables sql: " + sql) + sql = 'COPY "public"."party_party" TO STDOUT' + logger.error("experiment 20: SIMPLIFIED9 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED8 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED9 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: - logger.error("Real exception in SIMPLIFIED7: " + str(e)) + logger.error("Real exception in SIMPLIFIED9: " + str(e)) + + + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN " + start + " AND " + \ + end + " ORDER BY id) TO STDOUT" + + sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" + logger.error("experiment 20: SIMPLIFIED8 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 20: SIMPLIFIED8 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 20: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED8: " + str(e)) + + # Fails in next block try: cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 19: SIMPLIFIED7 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED7 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED7 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED7 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED7: " + str(e)) try: # Simple copy experiment sql = 'COPY "public"."election_election" TO STDOUT;' - logger.error("experiment 19: SIMPLIFIED6 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED6 retrieve_tables sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 19: SIMPLIFIED6: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED6: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") @@ -141,11 +171,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 19: SIMPLIFIED5 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED5 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED5 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED5 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED5: " + str(e)) @@ -153,11 +183,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 19: SIMPLIFIED4 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED4 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED4 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED4 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED4: " + str(e)) @@ -165,11 +195,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY public.candidate_candidatecampaign TO STDOUT" - logger.error("experiment 19: SIMPLIFIED3 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED3 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED3 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED3 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED3: " + str(e)) @@ -177,11 +207,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - logger.error("experiment 19: SIMPLIFIED2 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED2 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED2 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED2 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED2: " + str(e)) @@ -195,11 +225,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # end + " ORDER BY id) TO STDOUT" else: sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 19: SIMPLIFIED1 retrieve_tables sql: " + sql) + logger.error("experiment 20: SIMPLIFIED1 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: SIMPLIFIED1 after cur.copy_expert ") + logger.error("experiment 20: SIMPLIFIED1 after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED: " + str(e)) @@ -209,30 +239,30 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 19: file: " + str(file)) + logger.error("experiment 20: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 19: retrieve_tables sql: " + sql) + logger.error("experiment 20: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 19: after cur.copy_expert ") + logger.error("experiment 20: after cur.copy_expert ") file.seek(0) - logger.error("experiment 19: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 20: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 19: after file close, status " + status) + logger.error("experiment 20: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 19: after status +=, " + status) - logger.error("experiment 19: before conn.commit") + logger.error("experiment 20: after status +=, " + status) + logger.error("experiment 20: before conn.commit") conn.commit() - logger.error("experiment 19: after conn.commit ") + logger.error("experiment 20: after conn.commit ") conn.close() - logger.error("experiment 19: after conn.close ") + logger.error("experiment 20: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -242,14 +272,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 19: before results") + logger.error("experiment 20: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 19: results: " + str(results)) + logger.error("experiment 20: results: " + str(results)) return results except Exception as e: From 0081883dcde73644e06311a0afea01da2a881f69 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 15:16:05 -0700 Subject: [PATCH 48/98] Experiment 21, exploring copy_expert simplification --- retrieve_tables/controllers.py | 106 ++++++++++++++++++--------------- voter_guide/controllers.py | 26 ++++++-- 2 files changed, 78 insertions(+), 54 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index c5742f462..37d39d30b 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 20: psycopg2: " + line.strip()) + logger.error("experiment 21: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -90,54 +90,64 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: # Simple copy experiment sql = 'COPY "election_election" TO STDOUT;' - logger.error("experiment 20: SIMPLIFIED11 sql: " + sql) + logger.error("experiment 21: SIMPLIFIED12 sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 20: SIMPLIFIED11 select some stuff: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED12 select some stuff: " + file.readline().strip()) except Exception as e: - logger.error("Real exception in SIMPLIFIED11 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + logger.error("Real exception in SIMPLIFIED12 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") try: cur = conn.cursor() file = StringIO() # Empty file sql = 'COPY "party_party" TO STDOUT' - logger.error("experiment 20: SIMPLIFIED10 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED11 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED10 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED11 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED11 retrieve_tables file contents: " + file.readline().strip()) + except Exception as e: + logger.error("Real exception in SIMPLIFIED11: " + str(e)) + + try: + cur = conn.cursor() + file = StringIO() # Empty file + sql = 'COPY "public"."party_party" TO STDOUT' + logger.error("experiment 21: SIMPLIFIED10 retrieve_tables sql: " + sql) + cur.copy_expert(sql, file, size=8192) + logger.error("experiment 21: SIMPLIFIED10 after cur.copy_expert ") + file.seek(0) + logger.error("experiment 21: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED10: " + str(e)) + # Works to here try: cur = conn.cursor() file = StringIO() # Empty file sql = 'COPY "public"."party_party" TO STDOUT' - logger.error("experiment 20: SIMPLIFIED9 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED9 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED9 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED9 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED9: " + str(e)) - try: cur = conn.cursor() file = StringIO() # Empty file - sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN " + start + " AND " + \ - end + " ORDER BY id) TO STDOUT" + sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN 1 AND 1000 ORDER BY id) TO STDOUT" - sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 20: SIMPLIFIED8 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED8 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED8 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED8 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED8: " + str(e)) @@ -147,23 +157,23 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 20: SIMPLIFIED7 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED7 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED7 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED7 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED7: " + str(e)) try: # Simple copy experiment sql = 'COPY "public"."election_election" TO STDOUT;' - logger.error("experiment 20: SIMPLIFIED6 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED6 retrieve_tables sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 20: SIMPLIFIED6: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED6: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") @@ -171,11 +181,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 20: SIMPLIFIED5 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED5 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED5 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED5 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED5: " + str(e)) @@ -183,11 +193,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 20: SIMPLIFIED4 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED4 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED4 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED4 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED4: " + str(e)) @@ -195,11 +205,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY public.candidate_candidatecampaign TO STDOUT" - logger.error("experiment 20: SIMPLIFIED3 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED3 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED3 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED3 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED3: " + str(e)) @@ -207,11 +217,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - logger.error("experiment 20: SIMPLIFIED2 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED2 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED2 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED2 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED2: " + str(e)) @@ -225,11 +235,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # end + " ORDER BY id) TO STDOUT" else: sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 20: SIMPLIFIED1 retrieve_tables sql: " + sql) + logger.error("experiment 21: SIMPLIFIED1 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: SIMPLIFIED1 after cur.copy_expert ") + logger.error("experiment 21: SIMPLIFIED1 after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED: " + str(e)) @@ -239,30 +249,30 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 20: file: " + str(file)) + logger.error("experiment 21: file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 20: retrieve_tables sql: " + sql) + logger.error("experiment 21: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 20: after cur.copy_expert ") + logger.error("experiment 21: after cur.copy_expert ") file.seek(0) - logger.error("experiment 20: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 21: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 20: after file close, status " + status) + logger.error("experiment 21: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 20: after status +=, " + status) - logger.error("experiment 20: before conn.commit") + logger.error("experiment 21: after status +=, " + status) + logger.error("experiment 21: before conn.commit") conn.commit() - logger.error("experiment 20: after conn.commit ") + logger.error("experiment 21: after conn.commit ") conn.close() - logger.error("experiment 20: after conn.close ") + logger.error("experiment 21: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -272,14 +282,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 20: before results") + logger.error("experiment 21: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 20: results: " + str(results)) + logger.error("experiment 21: results: " + str(results)) return results except Exception as e: diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 0595ab90b..dbe4cff62 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -2217,19 +2217,33 @@ def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(vo voter_guide_possibility_id = voter_guide_possibility.id organization_we_vote_id = voter_guide_possibility.organization_we_vote_id voter_guide_possibility_url = voter_guide_possibility.voter_guide_possibility_url - url_root = voter_guide_possibility_url.split('/')[2] + parts = urlparse(voter_guide_possibility_url) + net_location = parts.netloc + print(net_location) - enddate = date.today() - startdate = enddate - timedelta(months=6) + enddate = datetime.now(pytz.UTC) + startdate = enddate - timedelta(days=186) # 6 months voter_guide_possibility_query = VoterGuidePossibility.objects.filter( - Q(voter_guide_possibility_url__contains=url_root) & + Q(voter_guide_possibility_url__contains=net_location) & Q(organization_we_vote_id__iexact=organization_we_vote_id) & - Q(date_last_changed__range=[startdate, enddate])).exclude(id=voter_guide_possibility_id) + Q(date_last_changed__range=[startdate, enddate])).order_by('-date_last_changed') + # leave destination in set .exclude(id=voter_guide_possibility_id) voter_guide_possibility_list = list(voter_guide_possibility_query) + ids_list = [] for voter_guide_possibility in voter_guide_possibility_list: + ids_list.append(voter_guide_possibility.id) + + possibility_position_query = VoterGuidePossibilityPosition.objects.filter( + Q(voter_guide_possibility_parent_id__in=ids_list)) + possibility_position_query_list = list(possibility_position_query) + newlist = sorted(possibility_position_query_list, key=lambda x: x.voter_guide_possibility_parent_id, reverse=True) + for blip in newlist: + print(str(blip)) + + # voter_guide_possibility_parent_id=voter_guide_possibility.id).order_by('possibility_position_number') # move all the latest voter_guide_possibility ies to this voter_guide_possibility_id as parent - print(voter_guide_possibility) + print(str(voter_guide_possibility)) # now itterate through this list, and move all the lastest_voter_guide_possibility ies to this voter_guide_possibility_id as parent From b1ac40637d2eb652cc314a215f9c45ae4430c95d Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Thu, 14 Jul 2022 15:53:12 -0700 Subject: [PATCH 49/98] Updated create_new_voter_account process to auto-verify emails entered by admin. If an account already exists, simply update with new permissions. If a voter approves a friend request, but doesn't have a verified email address attached to their profile, heal the data and mark the email_ownership_is_verified as true. --- apis_v1/views/views_voter.py | 52 +++++++++++++++++++++------ email_outbound/models.py | 9 +++-- friend/controllers.py | 68 +++++++++++++++++++++++++++++++++--- voter/models.py | 43 ++++++++++++++++++++--- 4 files changed, 150 insertions(+), 22 deletions(-) diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index fa7c3029f..f7fb35d8a 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -877,19 +877,51 @@ def voter_create_new_account_view(request): # voterCreateNewAccount is_verified_volunteer = request.GET.get('is_verified_volunteer', False) == 'true' # Check to make sure email isn't attached to existing account in EmailAddress table - email_address_queryset = EmailAddress.objects.all() - email_address_queryset = email_address_queryset.filter( - normalized_email_address__iexact=email, - deleted=False + existing_voter_found = False + email_manager = EmailManager() + voter = None + voter_manager = VoterManager() + existing_email_results = email_manager.retrieve_primary_email_with_ownership_verified( + normalized_email_address=email, ) - email_address_list = list(email_address_queryset) - email_already_in_use = True if len(email_address_list) > 0 else False - if email_already_in_use: - status += "EMAIL_ADDRESS_ALREADY_IN_USE " + if existing_email_results['email_address_object_found']: + email_address_object = existing_email_results['email_address_object'] + voter_we_vote_id = email_address_object.voter_we_vote_id + voter_results = voter_manager.retrieve_voter_by_we_vote_id(voter_we_vote_id) + if voter_results['voter_found']: + voter = voter_results['voter'] + existing_voter_found = True + else: + voter_results = voter_manager.retrieve_voter_by_email(email) + if voter_results['voter_found']: + voter = voter_results['voter'] + existing_voter_found = True + + if existing_voter_found: + voter.set_password(password) + if is_admin: + voter.is_admin = True + if is_analytics_admin: + voter.is_analytics_admin = True + if is_partner_organization: + voter.is_partner_organization = True + if is_political_data_manager: + voter.is_political_data_manager = True + if is_political_data_viewer: + voter.is_political_data_viewer = True + if is_verified_volunteer: + voter.is_verified_volunteer = True + if not positive_value_exists(voter.first_name) and positive_value_exists(first_name): + voter.first_name = first_name + if not positive_value_exists(voter.last_name) and positive_value_exists(last_name): + voter.last_name = last_name + voter.save() + + status += "EMAIL_ADDRESS_ALREADY_IN_USE-UPDATED_VOTER " json_data = { 'status': status, - 'success': False, - 'duplicate_email': True, + 'success': True, + 'duplicate_email': False, 'has_permission': True, } else: diff --git a/email_outbound/models.py b/email_outbound/models.py index 9c9d5faa1..4f05cdf86 100644 --- a/email_outbound/models.py +++ b/email_outbound/models.py @@ -457,8 +457,11 @@ def parse_raw_emails_into_list(self, email_addresses_raw): } return results - def retrieve_email_address_object(self, normalized_email_address='', email_address_object_we_vote_id='', - voter_we_vote_id=''): + def retrieve_email_address_object( + self, + normalized_email_address='', + email_address_object_we_vote_id='', + voter_we_vote_id=''): """ There are cases where we store multiple entries for the same normalized_email_address (prior to an email address being verified) @@ -728,7 +731,7 @@ def retrieve_voter_email_address_list(self, voter_we_vote_id): } return results - def retrieve_primary_email_with_ownership_verified(self, voter_we_vote_id, normalized_email_address=''): + def retrieve_primary_email_with_ownership_verified(self, voter_we_vote_id='', normalized_email_address=''): status = "" email_address_list = [] email_address_list_found = False diff --git a/friend/controllers.py b/friend/controllers.py index 880c61748..3b15043cd 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -1283,7 +1283,6 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify return error_results # Now that we have the friend_invitation data, look more closely at it - acceptance_email_should_be_sent = False invitation_found = False email_manager = EmailManager() if friend_invitation_results['friend_invitation_voter_link_found']: @@ -1306,11 +1305,72 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify } return error_results - # Now we want to make sure we have a current_friend entry + # voter_we_vote_id is probably NOT the same as voter_we_vote_id_accepting_invitation + # We want to make all changes against voter_we_vote_id_accepting_invitation, and those changes will be + # merged into the current voter_we_vote_id on a different API call recipient_organization_we_vote_id = '' voter_results = voter_manager.retrieve_voter_by_we_vote_id(voter_we_vote_id_accepting_invitation) - if voter_results['voter_found']: - recipient_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id + if not voter_results['voter_found']: + status += "VOTER_THIS_INVITATION_WAS_SENT_TO_COULD_NOT_BE_FOUND: " + voter_results['status'] + " " + error_results = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': True, + 'invitation_found': True, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + voter_accepting_invitation = voter_results['voter'] + recipient_organization_we_vote_id = voter_accepting_invitation.linked_organization_we_vote_id + + # Data healing may be needed + # Even though 'friend_invitation_voter_link_found', we don't necessarily have a previously verified email + # address. By clicking this link, voter_we_vote_id_accepting_invitation is verifying they have access + # to the email used to send this invite. + + # Since the 'friend_invitation_voter_link' doesn't include the email address used, we see if + # voter_accepting_invitation.email has been verified. If not we are going to verify that. + if voter_accepting_invitation.email and not voter_accepting_invitation.email_ownership_is_verified: + # Check to see if the email is owned by anyone else. If not, we are going to assume this is the + # email this voter just verified by clicking the friend invitation + temp_voter_we_vote_id = "" + email_results = email_manager.retrieve_primary_email_with_ownership_verified( + temp_voter_we_vote_id, voter_accepting_invitation.email) + if email_results['email_address_object_found']: + # The email belongs to this or another voter, and we don't want to proceed + # with any additional data healing + pass + else: + # If here, we know that voter_accepting_invitation.email hasn't been claimed by another account + # See if an email_address_object already exists for this voter + this_voter_email_results = email_manager.retrieve_email_address_object( + normalized_email_address=voter_accepting_invitation.email, + voter_we_vote_id=voter_we_vote_id_accepting_invitation) + email_address_object = None + email_address_object_found = False + if this_voter_email_results['email_address_object_found']: + email_address_object = this_voter_email_results['email_address_object'] + email_address_object_found = True + elif this_voter_email_results['email_address_list_found']: + email_address_list = this_voter_email_results['email_address_list'] + email_address_object = email_address_list[0] + email_address_object_found = True + if email_address_object_found: + try: + email_address_object.email_ownership_is_verified = True + email_address_object.save() + + voter_accepting_invitation.email_ownership_is_verified = True + voter_accepting_invitation.primary_email_we_vote_id = email_address_object.we_vote_id + voter_accepting_invitation.save() + except Exception as e: + status += "FAILED_TO_UPDATE_EMAIL_OWNERSHIP_IS_VERIFIED: " + str(e) + " " + + # Now we want to make sure we have a current_friend entry friend_results = friend_manager.update_or_create_current_friend( sender_voter_we_vote_id=sender_voter_we_vote_id, recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, diff --git a/voter/models.py b/voter/models.py index 0c3fdcc07..55d413927 100644 --- a/voter/models.py +++ b/voter/models.py @@ -710,8 +710,11 @@ def create_new_voter_account(self, first_name, last_name, email, password, is_ad """ voter = Voter() success = False - status = "Failed to create voter" + status = "" duplicate_email = False + email_address_object_created = False + email_address_we_vote_id = '' + voter_created = False try: voter.set_password(password) voter.first_name = first_name @@ -725,21 +728,51 @@ def create_new_voter_account(self, first_name, last_name, email, password, is_ad voter.is_verified_volunteer = is_verified_volunteer voter.is_active = True voter.save() + voter_created = True success = True - status = "Created voter " + voter.we_vote_id + status = "Created voter " + str(voter.we_vote_id) + " " logger.debug("create_new_voter_account successfully created (voter) : " + first_name) - except IntegrityError as e: - status += ", " + str(e) + status += "FAILED_TO_CREATE_VOTER_INTEGRITY: " + str(e) + " " handle_record_not_saved_exception(e, logger=logger) print("create_new_voter_account IntegrityError exception:" + str(e)) if "voter_voter_email_key" in str(e): duplicate_email = True except Exception as e: - status += ", " + str(e) + status += "FAILED_TO_CREATE_VOTER: " + str(e) + " " handle_record_not_saved_exception(e, logger=logger) logger.debug("create_new_voter_account general exception: " + str(e)) + if voter_created: + try: + from email_outbound.models import EmailManager + email_manager = EmailManager() + email_results = email_manager.create_email_address( + normalized_email_address=email, + voter_we_vote_id=voter.we_vote_id, + email_ownership_is_verified=True, + ) + email_address_object_created = True + status += email_results['status'] + if email_results['email_address_object_saved']: + email_address_object = email_results['email_address_object'] + email_address_we_vote_id = email_address_object.we_vote_id + except Exception as e: + status += "FAILED_TO_CREATE_VOTER_EMAIL_ADDRESS: " + str(e) + " " + handle_record_not_saved_exception(e, logger=logger) + logger.debug("ERROR_SAVING_NEW_EMAIL create_new_voter_account general exception: " + str(e)) + + if email_address_object_created: + try: + voter.email_ownership_is_verified = True + voter.primary_email_we_vote_id = email_address_we_vote_id + voter.save() + status += "VOTER_CREATED_EMAIL_OWNERSHIP_VERIFIED " + except Exception as e: + status += "FAILED_TO_UPDATE_VOTER_WITH_EMAIL_ADDRESS: " + str(e) + " " + handle_record_not_saved_exception(e, logger=logger) + logger.debug("ERROR_SAVING_NEW_EMAIL create_new_voter_account general exception: " + str(e)) + results = { 'success': success, 'status': status, From 56a33d02c5f892d86bfb2a3dd070d060d77a36fe Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 16:23:07 -0700 Subject: [PATCH 50/98] Experiment 22, exploring copy_expert simplification --- retrieve_tables/controllers.py | 232 ++++++++++++++++----------------- voter_guide/controllers.py | 139 +++----------------- 2 files changed, 137 insertions(+), 234 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 37d39d30b..ea1a7889c 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -23,7 +23,6 @@ # 'campaign_campaignx_politician', # 'campaign_campaignxseofriendlypath', allowable_tables = [ - 'candidate_candidatecampaign', 'candidate_candidatesarenotduplicates', 'candidate_candidatetoofficelink', 'elected_office_electedoffice', @@ -54,6 +53,7 @@ 'wevote_settings_wevotesetting', 'ballot_ballotitem', 'ballot_ballotreturned', + 'candidate_candidatecampaign', # 7/14/22 ... table possibly corrupted in AWS, so running it last ] dummy_unique_id = 10000000 @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 21: psycopg2: " + line.strip()) + logger.error("experiment 22: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -90,12 +90,12 @@ def retrieve_sql_tables_as_csv(table_name, start, end): try: # Simple copy experiment sql = 'COPY "election_election" TO STDOUT;' - logger.error("experiment 21: SIMPLIFIED12 sql: " + sql) + logger.error("experiment 22: SIMPLIFIED12 sql: " + sql) file = StringIO() # Empty file cur = conn.cursor() cur.copy_expert(sql, file, size=8192) file.seek(0) - logger.error("experiment 21: SIMPLIFIED12 select some stuff: " + file.readline().strip()) + logger.error("experiment 22: SIMPLIFIED12 select some stuff: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED12 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") @@ -103,11 +103,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = 'COPY "party_party" TO STDOUT' - logger.error("experiment 21: SIMPLIFIED11 retrieve_tables sql: " + sql) + logger.error("experiment 22: SIMPLIFIED11 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED11 after cur.copy_expert ") + logger.error("experiment 22: SIMPLIFIED11 after cur.copy_expert ") file.seek(0) - logger.error("experiment 21: SIMPLIFIED11 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 22: SIMPLIFIED11 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED11: " + str(e)) @@ -115,11 +115,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = 'COPY "public"."party_party" TO STDOUT' - logger.error("experiment 21: SIMPLIFIED10 retrieve_tables sql: " + sql) + logger.error("experiment 22: SIMPLIFIED10 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED10 after cur.copy_expert ") + logger.error("experiment 22: SIMPLIFIED10 after cur.copy_expert ") file.seek(0) - logger.error("experiment 21: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 22: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED10: " + str(e)) @@ -130,11 +130,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file sql = 'COPY "public"."party_party" TO STDOUT' - logger.error("experiment 21: SIMPLIFIED9 retrieve_tables sql: " + sql) + logger.error("experiment 22: SIMPLIFIED9 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED9 after cur.copy_expert ") + logger.error("experiment 22: SIMPLIFIED9 after cur.copy_expert ") file.seek(0) - logger.error("experiment 21: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 22: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED9: " + str(e)) @@ -143,105 +143,105 @@ def retrieve_sql_tables_as_csv(table_name, start, end): file = StringIO() # Empty file sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN 1 AND 1000 ORDER BY id) TO STDOUT" - logger.error("experiment 21: SIMPLIFIED8 retrieve_tables sql: " + sql) + logger.error("experiment 22: SIMPLIFIED8 retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED8 after cur.copy_expert ") + logger.error("experiment 22: SIMPLIFIED8 after cur.copy_expert ") file.seek(0) - logger.error("experiment 21: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 22: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) except Exception as e: logger.error("Real exception in SIMPLIFIED8: " + str(e)) # Fails in next block - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 21: SIMPLIFIED7 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED7 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 21: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED7: " + str(e)) - - try: - # Simple copy experiment - sql = 'COPY "public"."election_election" TO STDOUT;' - logger.error("experiment 21: SIMPLIFIED6 retrieve_tables sql: " + sql) - file = StringIO() # Empty file - cur = conn.cursor() - cur.copy_expert(sql, file, size=8192) - file.seek(0) - logger.error("experiment 21: SIMPLIFIED6: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 21: SIMPLIFIED5 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED5 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 21: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED5: " + str(e)) - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - logger.error("experiment 21: SIMPLIFIED4 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED4 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 21: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED4: " + str(e)) - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = "COPY public.candidate_candidatecampaign TO STDOUT" - logger.error("experiment 21: SIMPLIFIED3 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED3 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 21: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED3: " + str(e)) - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - logger.error("experiment 21: SIMPLIFIED2 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED2 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 21: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED2: " + str(e)) - - try: - cur = conn.cursor() - file = StringIO() # Empty file - if positive_value_exists(end): - sql = "COPY (SELECT * FROM \"public\".\"" + table_name + "\" WHERE id BETWEEN " + start + " AND " + \ - end + " ORDER BY id) TO STDOUT" - # sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ - # end + " ORDER BY id) TO STDOUT" - else: - sql = "COPY " + table_name + " TO STDOUT" - logger.error("experiment 21: SIMPLIFIED1 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: SIMPLIFIED1 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 21: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED: " + str(e)) + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" + # logger.error("experiment 22: SIMPLIFIED7 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 22: SIMPLIFIED7 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED7: " + str(e)) + # + # try: + # # Simple copy experiment + # sql = 'COPY "public"."election_election" TO STDOUT;' + # logger.error("experiment 22: SIMPLIFIED6 retrieve_tables sql: " + sql) + # file = StringIO() # Empty file + # cur = conn.cursor() + # cur.copy_expert(sql, file, size=8192) + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED6: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" + # logger.error("experiment 22: SIMPLIFIED5 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 22: SIMPLIFIED5 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED5: " + str(e)) + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" + # logger.error("experiment 22: SIMPLIFIED4 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 22: SIMPLIFIED4 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED4: " + str(e)) + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = "COPY public.candidate_candidatecampaign TO STDOUT" + # logger.error("experiment 22: SIMPLIFIED3 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 22: SIMPLIFIED3 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED3: " + str(e)) + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" + # logger.error("experiment 22: SIMPLIFIED2 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 22: SIMPLIFIED2 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED2: " + str(e)) + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # if positive_value_exists(end): + # sql = "COPY (SELECT * FROM \"public\".\"" + table_name + "\" WHERE id BETWEEN " + start + " AND " + \ + # end + " ORDER BY id) TO STDOUT" + # # sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + # # end + " ORDER BY id) TO STDOUT" + # else: + # sql = "COPY " + table_name + " TO STDOUT" + # logger.error("experiment 22: SIMPLIFIED1 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 22: SIMPLIFIED1 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 22: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED: " + str(e)) csv_files = {} if table_name in allowable_tables: @@ -249,30 +249,30 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 21: file: " + str(file)) + logger.error("experiment 22: REAL FILE LOOP FOR file: " + str(file)) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 21: retrieve_tables sql: " + sql) + logger.error("experiment 22: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 21: after cur.copy_expert ") + logger.error("experiment 22: after cur.copy_expert ") file.seek(0) - logger.error("experiment 21: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 22: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 21: after file close, status " + status) + logger.error("experiment 22: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 21: after status +=, " + status) - logger.error("experiment 21: before conn.commit") + logger.error("experiment 22: after status +=, " + status) + logger.error("experiment 22: before conn.commit") conn.commit() - logger.error("experiment 21: after conn.commit ") + logger.error("experiment 22: after conn.commit ") conn.close() - logger.error("experiment 21: after conn.close ") + logger.error("experiment 22: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -282,14 +282,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 21: before results") + logger.error("experiment 22: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 21: results: " + str(results)) + logger.error("experiment 22: results: " + str(results)) return results except Exception as e: diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index dbe4cff62..985fb072f 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -2116,7 +2116,7 @@ def voter_guide_possibility_positions_retrieve_for_api( # voterGuidePossibility voter_guide_possibility_id=voter_guide_possibility_id) # TODO: Steve, move all the voterGuidPossiblityPositions with the same organization_we_vote_id that was changed in the last 6 months - move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(results['voter_guide_possibility']) + move_voter_guide_possibility_positions_to_requested_voter_guide_possibility(results['voter_guide_possibility']) possible_endorsement_list = [] @@ -2213,7 +2213,7 @@ def voter_guide_possibility_positions_retrieve_for_api( # voterGuidePossibility return json_data -def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(voter_guide_possibility): +def move_voter_guide_possibility_positions_to_requested_voter_guide_possibility(voter_guide_possibility): voter_guide_possibility_id = voter_guide_possibility.id organization_we_vote_id = voter_guide_possibility.organization_we_vote_id voter_guide_possibility_url = voter_guide_possibility.voter_guide_possibility_url @@ -2235,123 +2235,26 @@ def move_voter_guide_possibility_positions_to_lastest_voter_guide_possibility(vo ids_list.append(voter_guide_possibility.id) possibility_position_query = VoterGuidePossibilityPosition.objects.filter( - Q(voter_guide_possibility_parent_id__in=ids_list)) + Q(voter_guide_possibility_parent_id__in=ids_list)).order_by('ballot_item_name', '-voter_guide_possibility_parent_id') possibility_position_query_list = list(possibility_position_query) - newlist = sorted(possibility_position_query_list, key=lambda x: x.voter_guide_possibility_parent_id, reverse=True) - for blip in newlist: - print(str(blip)) - - # voter_guide_possibility_parent_id=voter_guide_possibility.id).order_by('possibility_position_number') - # move all the latest voter_guide_possibility ies to this voter_guide_possibility_id as parent - print(str(voter_guide_possibility)) - - # now itterate through this list, and move all the lastest_voter_guide_possibility ies to this voter_guide_possibility_id as parent - - # voter_guide_possibility_query = voter_guide_possibility_query.filter(date_last_changed__year=now.year) - # # DALE 2020-06-08 After working with this, it is better to include entries hidden from active review - # # voter_guide_possibility_query = voter_guide_possibility_query.exclude(hide_from_active_review=True) - # - # # Only retrieve by URL if it was created this year - # own = datetime.now() - # status += "LIMITING_TO_THIS_YEAR: " + str(now.year) + " " - # - # voter_guide_possibility_on_stage = voter_guide_possibility_query.last() - # if voter_guide_possibility_on_stage is not None: - # voter_guide_possibility_on_stage_id = voter_guide_possibility_on_stage.id - # status += "VOTER_GUIDE_POSSIBILITY_FOUND_WITH_URL " - # success = True - # else: - # status += "VOTER_GUIDE_POSSIBILITY_NOT_FOUND_WITH_URL " - # success = True - # - # possibility_position_query = VoterGuidePossibilityPosition.objects.filter( - # voter_guide_possibility_parent_id=voter_guide_possibility.id).order_by('possibility_position_number') - # possibility_position_list = list(possibility_position_query) - # for possibility_position in possibility_position_list: - # if positive_value_exists(possibility_position.more_info_url): - # more_info_url = possibility_position.more_info_url - # else: - # more_info_url = voter_guide_possibility.voter_guide_possibility_url - # if voter_guide_possibility.voter_guide_possibility_type == ORGANIZATION_ENDORSING_CANDIDATES \ - # or voter_guide_possibility.voter_guide_possibility_type == UNKNOWN_TYPE: - # # The organization variables have been set above - # # Note that UNKNOWN_TYPE might be set if we are looking at organization - # # ###################### - # # If we are starting from a single organization endorsing many candidates, - # # we store that candidate information once - # if positive_value_exists(possibility_position.candidate_we_vote_id): - # candidate_name = possibility_position.ballot_item_name - # position_json = { - # 'candidate_name': candidate_name, - # 'candidate_we_vote_id': possibility_position.candidate_we_vote_id, - # 'candidate_twitter_handle': possibility_position.candidate_twitter_handle, - # 'google_civic_election_id': possibility_position.google_civic_election_id, - # 'more_info_url': more_info_url, - # 'organization_name': organization_name, - # 'organization_we_vote_id': organization_we_vote_id, - # 'organization_twitter_handle': organization_twitter_handle, - # 'stance': possibility_position.position_stance, - # 'statement_text': possibility_position.statement_text, - # 'state_code': state_code, - # } - # position_json_list.append(position_json) - # elif positive_value_exists(possibility_position.measure_we_vote_id): - # measure_title = possibility_position.ballot_item_name - # position_json = { - # 'google_civic_election_id': possibility_position.google_civic_election_id, - # 'measure_title': measure_title, - # 'measure_we_vote_id': possibility_position.measure_we_vote_id, - # 'more_info_url': more_info_url, - # 'organization_name': organization_name, - # 'organization_we_vote_id': organization_we_vote_id, - # 'organization_twitter_handle': organization_twitter_handle, - # 'stance': possibility_position.position_stance, - # 'statement_text': possibility_position.statement_text, - # 'state_code': state_code, - # } - # position_json_list.append(position_json) - # else: - # status += "MISSING_BOTH_CANDIDATE_AND_MEASURE_WE_VOTE_ID " - # elif voter_guide_possibility.voter_guide_possibility_type == ENDORSEMENTS_FOR_CANDIDATE: - # # ###################### - # # If we are starting from a single candidate endorsed by many "organizations" (which may be people), - # # we store the unique organization information in the VoterGuidePossibilityPosition table - # organization_name = possibility_position.organization_name \ - # if positive_value_exists(possibility_position.organization_name) else "" - # organization_we_vote_id = possibility_position.organization_we_vote_id \ - # if positive_value_exists(possibility_position.organization_we_vote_id) else "" - # organization_twitter_handle = possibility_position.organization_twitter_handle \ - # if positive_value_exists(possibility_position.organization_twitter_handle) else "" - # position_json = { - # 'candidate_name': candidate_name, - # 'candidate_we_vote_id': candidate_we_vote_id, - # 'candidate_twitter_handle': candidate_twitter_handle, - # 'contest_office_name': contest_office_name, - # 'google_civic_election_id': possibility_position.google_civic_election_id, - # 'measure_we_vote_id': possibility_position.measure_we_vote_id, - # 'more_info_url': more_info_url, - # 'organization_name': organization_name, - # 'organization_we_vote_id': organization_we_vote_id, - # 'organization_twitter_handle': organization_twitter_handle, - # 'stance': possibility_position.position_stance, - # 'statement_text': possibility_position.statement_text, - # 'state_code': state_code, - # } - # position_json_list.append(position_json) - # else: - # # This is an error condition which should not be reached - # status += "UNRECOGNIZED_VOTER_POSSIBILITY_TYPE " - # - # if len(position_json_list): - # position_json_list_found = True - # - # results = { - # 'status': status, - # 'success': success, - # 'position_json_list': position_json_list, - # 'position_json_list_found': position_json_list_found, - # } - # return results + for blip in possibility_position_query_list: + print(str(blip.ballot_item_name) + " " + str(blip.voter_guide_possibility_parent_id)) + + # Remove duplicates, for now if the voter_guide_possibility_parent_id numer is higher, that is the one we keep + seen_candidates = set() + new_list = [] + for position in possibility_position_query_list: + if position.ballot_item_name not in seen_candidates: + new_list.append(position) + seen_candidates.add(position.ballot_item_name) + + print('-------') + for blip in new_list: + print(str(blip.ballot_item_name) + " " + str(blip.voter_guide_possibility_parent_id)) + print('-------') + + results = {} + return results From 10434418124c617a388e9f8313d422d0abc00362 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 16:45:42 -0700 Subject: [PATCH 51/98] Experiment 23, exploring copy_expert simplification Suspect that there are problems with 'organization_organization', and 'candidate_candidatecampaign', so running them last, and seeing how far we get --- retrieve_tables/controllers.py | 184 ++++++++++++++++----------------- voter_guide/controllers.py | 8 ++ 2 files changed, 100 insertions(+), 92 deletions(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index ea1a7889c..02b986ee4 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -39,7 +39,6 @@ 'office_contestoffice', 'office_contestofficesarenotduplicates', 'office_contestofficevisitingotherelection', - 'organization_organization', 'organization_organizationreserveddomain', 'party_party', 'politician_politician', @@ -53,6 +52,7 @@ 'wevote_settings_wevotesetting', 'ballot_ballotitem', 'ballot_ballotreturned', + 'organization_organization', # 7/14/22 ... table possibly corrupted in AWS, so running it last 'candidate_candidatecampaign', # 7/14/22 ... table possibly corrupted in AWS, so running it last ] @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 22: psycopg2: " + line.strip()) + logger.error("experiment 23: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -87,69 +87,69 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # logger.debug("retrieve_sql_tables_as_csv psycopg2 Connected to DB") - try: - # Simple copy experiment - sql = 'COPY "election_election" TO STDOUT;' - logger.error("experiment 22: SIMPLIFIED12 sql: " + sql) - file = StringIO() # Empty file - cur = conn.cursor() - cur.copy_expert(sql, file, size=8192) - file.seek(0) - logger.error("experiment 22: SIMPLIFIED12 select some stuff: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED12 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = 'COPY "party_party" TO STDOUT' - logger.error("experiment 22: SIMPLIFIED11 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 22: SIMPLIFIED11 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 22: SIMPLIFIED11 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED11: " + str(e)) - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = 'COPY "public"."party_party" TO STDOUT' - logger.error("experiment 22: SIMPLIFIED10 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 22: SIMPLIFIED10 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 22: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED10: " + str(e)) + # try: + # # Simple copy experiment + # sql = 'COPY "election_election" TO STDOUT;' + # logger.error("experiment 23: SIMPLIFIED12 sql: " + sql) + # file = StringIO() # Empty file + # cur = conn.cursor() + # cur.copy_expert(sql, file, size=8192) + # file.seek(0) + # logger.error("experiment 23: SIMPLIFIED12 select some stuff: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED12 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = 'COPY "party_party" TO STDOUT' + # logger.error("experiment 23: SIMPLIFIED11 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 23: SIMPLIFIED11 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 23: SIMPLIFIED11 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED11: " + str(e)) + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = 'COPY "public"."party_party" TO STDOUT' + # logger.error("experiment 23: SIMPLIFIED10 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 23: SIMPLIFIED10 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 23: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED10: " + str(e)) # Works to here - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = 'COPY "public"."party_party" TO STDOUT' - logger.error("experiment 22: SIMPLIFIED9 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 22: SIMPLIFIED9 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 22: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED9: " + str(e)) - - try: - cur = conn.cursor() - file = StringIO() # Empty file - sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN 1 AND 1000 ORDER BY id) TO STDOUT" - - logger.error("experiment 22: SIMPLIFIED8 retrieve_tables sql: " + sql) - cur.copy_expert(sql, file, size=8192) - logger.error("experiment 22: SIMPLIFIED8 after cur.copy_expert ") - file.seek(0) - logger.error("experiment 22: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) - except Exception as e: - logger.error("Real exception in SIMPLIFIED8: " + str(e)) + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = 'COPY "public"."party_party" TO STDOUT' + # logger.error("experiment 23: SIMPLIFIED9 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 23: SIMPLIFIED9 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 23: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED9: " + str(e)) + # + # try: + # cur = conn.cursor() + # file = StringIO() # Empty file + # sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN 1 AND 1000 ORDER BY id) TO STDOUT" + # + # logger.error("experiment 23: SIMPLIFIED8 retrieve_tables sql: " + sql) + # cur.copy_expert(sql, file, size=8192) + # logger.error("experiment 23: SIMPLIFIED8 after cur.copy_expert ") + # file.seek(0) + # logger.error("experiment 23: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) + # except Exception as e: + # logger.error("Real exception in SIMPLIFIED8: " + str(e)) # Fails in next block @@ -157,23 +157,23 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # cur = conn.cursor() # file = StringIO() # Empty file # sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - # logger.error("experiment 22: SIMPLIFIED7 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED7 retrieve_tables sql: " + sql) # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 22: SIMPLIFIED7 after cur.copy_expert ") + # logger.error("experiment 23: SIMPLIFIED7 after cur.copy_expert ") # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED7: " + str(e)) # # try: # # Simple copy experiment # sql = 'COPY "public"."election_election" TO STDOUT;' - # logger.error("experiment 22: SIMPLIFIED6 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED6 retrieve_tables sql: " + sql) # file = StringIO() # Empty file # cur = conn.cursor() # cur.copy_expert(sql, file, size=8192) # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED6: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED6: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") # @@ -181,11 +181,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # cur = conn.cursor() # file = StringIO() # Empty file # sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - # logger.error("experiment 22: SIMPLIFIED5 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED5 retrieve_tables sql: " + sql) # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 22: SIMPLIFIED5 after cur.copy_expert ") + # logger.error("experiment 23: SIMPLIFIED5 after cur.copy_expert ") # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED5: " + str(e)) # @@ -193,11 +193,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # cur = conn.cursor() # file = StringIO() # Empty file # sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - # logger.error("experiment 22: SIMPLIFIED4 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED4 retrieve_tables sql: " + sql) # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 22: SIMPLIFIED4 after cur.copy_expert ") + # logger.error("experiment 23: SIMPLIFIED4 after cur.copy_expert ") # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED4: " + str(e)) # @@ -205,11 +205,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # cur = conn.cursor() # file = StringIO() # Empty file # sql = "COPY public.candidate_candidatecampaign TO STDOUT" - # logger.error("experiment 22: SIMPLIFIED3 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED3 retrieve_tables sql: " + sql) # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 22: SIMPLIFIED3 after cur.copy_expert ") + # logger.error("experiment 23: SIMPLIFIED3 after cur.copy_expert ") # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED3: " + str(e)) # @@ -217,11 +217,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # cur = conn.cursor() # file = StringIO() # Empty file # sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - # logger.error("experiment 22: SIMPLIFIED2 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED2 retrieve_tables sql: " + sql) # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 22: SIMPLIFIED2 after cur.copy_expert ") + # logger.error("experiment 23: SIMPLIFIED2 after cur.copy_expert ") # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED2: " + str(e)) # @@ -235,11 +235,11 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # # end + " ORDER BY id) TO STDOUT" # else: # sql = "COPY " + table_name + " TO STDOUT" - # logger.error("experiment 22: SIMPLIFIED1 retrieve_tables sql: " + sql) + # logger.error("experiment 23: SIMPLIFIED1 retrieve_tables sql: " + sql) # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 22: SIMPLIFIED1 after cur.copy_expert ") + # logger.error("experiment 23: SIMPLIFIED1 after cur.copy_expert ") # file.seek(0) - # logger.error("experiment 22: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) + # logger.error("experiment 23: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) # except Exception as e: # logger.error("Real exception in SIMPLIFIED: " + str(e)) @@ -249,30 +249,30 @@ def retrieve_sql_tables_as_csv(table_name, start, end): cur = conn.cursor() file = StringIO() # Empty file - logger.error("experiment 22: REAL FILE LOOP FOR file: " + str(file)) + # logger.error("experiment 23: REAL FILE LOOP FOR file: " + table_name) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 22: retrieve_tables sql: " + sql) + logger.error("experiment 23: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 22: after cur.copy_expert ") + logger.error("experiment 23: after cur.copy_expert ") file.seek(0) - logger.error("experiment 22: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 23: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 22: after file close, status " + status) + logger.error("experiment 23: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 22: after status +=, " + status) - logger.error("experiment 22: before conn.commit") + logger.error("experiment 23: after status +=, " + status) + logger.error("experiment 23: before conn.commit") conn.commit() - logger.error("experiment 22: after conn.commit ") + logger.error("experiment 23: after conn.commit ") conn.close() - logger.error("experiment 22: after conn.close ") + logger.error("experiment 23: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) @@ -282,14 +282,14 @@ def retrieve_sql_tables_as_csv(table_name, start, end): status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 22: before results") + logger.error("experiment 23: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 22: results: " + str(results)) + logger.error("experiment 23: results: " + str(results)) return results except Exception as e: diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index 985fb072f..edfd2a5e4 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -2247,6 +2247,14 @@ def move_voter_guide_possibility_positions_to_requested_voter_guide_possibility( if position.ballot_item_name not in seen_candidates: new_list.append(position) seen_candidates.add(position.ballot_item_name) + # TODO: If we have to change all the exising VoterGuidePossibilityPosition entries to match the current + # query, our design could use improvement + if position.voter_guide_possibility_parent_id != voter_guide_possibility_id: + # position.voter_guide_possibility_parent_id = voter_guide_possibility_id + logger.log("Updating '" + position.ballot_item_name + "' from parent id " + + position.voter_guide_possibility_parent_id + " to " + + voter_guide_possibility_id) + # position.save() print('-------') for blip in new_list: From f2e4f19aabecbbff859d6f0729b2b5b39e03c6b8 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 19:36:13 -0700 Subject: [PATCH 52/98] Experiment 24, exploring copy_expert simplification Suspect that there are problems with these tables in the AWS postgres db: 'organization_organization' 'candidate_candidatecampaign' 'polling_location_pollinglocation' so running them last, and seeing how far we get --- friend/models.py | 1 - retrieve_tables/controllers.py | 193 ++++----------------------------- voter_guide/controllers.py | 9 +- 3 files changed, 25 insertions(+), 178 deletions(-) diff --git a/friend/models.py b/friend/models.py index 17636ebf9..28a1312eb 100644 --- a/friend/models.py +++ b/friend/models.py @@ -1799,7 +1799,6 @@ def fetch_voters_with_friends_dataset_improved(self): port=get_environment_variable('DATABASE_PORT') ) cur = conn.cursor() - # July 14, 2022: If this works locally but not in AWS, then consider dropping the "public". part sql_viewer = \ 'SELECT id, COUNT(*) AS we_count FROM ( ' \ 'SELECT viewee_voter_we_vote_id id FROM "public"."friend_currentfriend" ' \ diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 02b986ee4..c227f78d3 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -43,7 +43,6 @@ 'party_party', 'politician_politician', 'politician_politiciansarenotduplicates', - 'polling_location_pollinglocation', 'position_positionentered', 'twitter_twitterlinktoorganization', 'voter_guide_voterguidepossibility', @@ -52,8 +51,9 @@ 'wevote_settings_wevotesetting', 'ballot_ballotitem', 'ballot_ballotreturned', - 'organization_organization', # 7/14/22 ... table possibly corrupted in AWS, so running it last - 'candidate_candidatecampaign', # 7/14/22 ... table possibly corrupted in AWS, so running it last + 'polling_location_pollinglocation', # 7/14/22 ... table possibly corrupted in AWS, so running it last + 'organization_organization', # 7/14/22 ... table possibly corrupted in AWS, so running it last + 'candidate_candidatecampaign', # 7/14/22 ... table possibly corrupted in AWS, so running it last ] dummy_unique_id = 10000000 @@ -74,7 +74,7 @@ def retrieve_sql_tables_as_csv(table_name, start, end): f = open("requirements.txt", "r") for line in f: if "psycopg2" in line: - logger.error("experiment 23: psycopg2: " + line.strip()) + logger.error("experiment 24: psycopg2: " + line.strip()) try: conn = psycopg2.connect( @@ -87,211 +87,60 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # logger.debug("retrieve_sql_tables_as_csv psycopg2 Connected to DB") - # try: - # # Simple copy experiment - # sql = 'COPY "election_election" TO STDOUT;' - # logger.error("experiment 23: SIMPLIFIED12 sql: " + sql) - # file = StringIO() # Empty file - # cur = conn.cursor() - # cur.copy_expert(sql, file, size=8192) - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED12 select some stuff: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED12 select some stuff retrieve_sql_tables_as_csv(): " + str(e) + " ") - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = 'COPY "party_party" TO STDOUT' - # logger.error("experiment 23: SIMPLIFIED11 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED11 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED11 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED11: " + str(e)) - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = 'COPY "public"."party_party" TO STDOUT' - # logger.error("experiment 23: SIMPLIFIED10 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED10 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED10 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED10: " + str(e)) - - - # Works to here - - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = 'COPY "public"."party_party" TO STDOUT' - # logger.error("experiment 23: SIMPLIFIED9 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED9 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED9 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED9: " + str(e)) - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = "COPY (SELECT * FROM \"public\".\"party_party\" WHERE id BETWEEN 1 AND 1000 ORDER BY id) TO STDOUT" - # - # logger.error("experiment 23: SIMPLIFIED8 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED8 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED8 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED8: " + str(e)) - - # Fails in next block - - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = "COPY \"candidate_candidatecampaign\" TO STDOUT" - # logger.error("experiment 23: SIMPLIFIED7 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED7 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED7 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED7: " + str(e)) - # - # try: - # # Simple copy experiment - # sql = 'COPY "public"."election_election" TO STDOUT;' - # logger.error("experiment 23: SIMPLIFIED6 retrieve_tables sql: " + sql) - # file = StringIO() # Empty file - # cur = conn.cursor() - # cur.copy_expert(sql, file, size=8192) - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED6: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED6: " + str(e) + " ") - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - # logger.error("experiment 23: SIMPLIFIED5 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED5 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED5 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED5: " + str(e)) - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = "COPY \"public\".\"candidate_candidatecampaign\" TO STDOUT" - # logger.error("experiment 23: SIMPLIFIED4 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED4 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED4 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED4: " + str(e)) - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = "COPY public.candidate_candidatecampaign TO STDOUT" - # logger.error("experiment 23: SIMPLIFIED3 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED3 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED3 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED3: " + str(e)) - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # sql = "COPY (SELECT * FROM public.candidate_candidatecampaign) TO STDOUT" - # logger.error("experiment 23: SIMPLIFIED2 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED2 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED2 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED2: " + str(e)) - # - # try: - # cur = conn.cursor() - # file = StringIO() # Empty file - # if positive_value_exists(end): - # sql = "COPY (SELECT * FROM \"public\".\"" + table_name + "\" WHERE id BETWEEN " + start + " AND " + \ - # end + " ORDER BY id) TO STDOUT" - # # sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ - # # end + " ORDER BY id) TO STDOUT" - # else: - # sql = "COPY " + table_name + " TO STDOUT" - # logger.error("experiment 23: SIMPLIFIED1 retrieve_tables sql: " + sql) - # cur.copy_expert(sql, file, size=8192) - # logger.error("experiment 23: SIMPLIFIED1 after cur.copy_expert ") - # file.seek(0) - # logger.error("experiment 23: SIMPLIFIED1 retrieve_tables file contents: " + file.readline().strip()) - # except Exception as e: - # logger.error("Real exception in SIMPLIFIED: " + str(e)) - csv_files = {} if table_name in allowable_tables: try: cur = conn.cursor() file = StringIO() # Empty file - # logger.error("experiment 23: REAL FILE LOOP FOR file: " + table_name) + logger.error("experiment 24: REAL FILE ALLOWED FOR file: " + table_name) if positive_value_exists(end): sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" - logger.error("experiment 23: retrieve_tables sql: " + sql) + logger.error("experiment 24: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("experiment 23: after cur.copy_expert ") + logger.error("experiment 24: after cur.copy_expert ") file.seek(0) - logger.error("experiment 23: retrieve_tables file contents: " + file.readline().strip()) + logger.error("experiment 24: retrieve_tables file contents: " + file.readline().strip()) file.seek(0) csv_files[table_name] = file.read() file.close() - logger.error("experiment 23: after file close, status " + status) + logger.error("experiment 24: after file close, status " + status) if "exported" not in status: status += "exported " status += table_name + "(" + start + "," + end + "), " - logger.error("experiment 23: after status +=, " + status) - logger.error("experiment 23: before conn.commit") + logger.error("experiment 24: after status +=, " + status) + logger.error("experiment 24: before conn.commit") conn.commit() - logger.error("experiment 23: after conn.commit ") + logger.error("experiment 24: after conn.commit ") conn.close() - logger.error("experiment 23: after conn.close ") + logger.error("experiment 24: after conn.close ") dt = time.time() - t0 logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + ' seconds. start = ' + start + ', end = ' + end) except Exception as e: - logger.error("Real exception in retrieve_sql_tables_as_csv(): " + str(e) + " ") + logger.error("experiment 24: Real exception in retrieve_sql_tables_as_csv(): " + str(e) + " ") else: status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) - logger.error("experiment 23: before results") + logger.error("experiment 24: before results") results = { 'success': True, 'status': status, 'files': csv_files, } - logger.error("experiment 23: results: " + str(results)) + logger.error("experiment 24: results: " + str(results)) return results + # July 2022: Unfortunately psycopg2-binary crashes and brings down the python thread hard, with nothing in the + # AWS log if a postgres file gets corrupted. + # run `pg_dump -f /dev/null WeVoteServerDB` from a terminal prompt, + # "This will read all tables and throw an error at the first corruption that causes an error." + # `pg_dump -f /dev/null wevotedev` on the server except Exception as e: status += "retrieve_tables export_sync_files_to_csv caught " + str(e) logger.error(status) diff --git a/voter_guide/controllers.py b/voter_guide/controllers.py index edfd2a5e4..2c9b4c0bc 100644 --- a/voter_guide/controllers.py +++ b/voter_guide/controllers.py @@ -2247,13 +2247,12 @@ def move_voter_guide_possibility_positions_to_requested_voter_guide_possibility( if position.ballot_item_name not in seen_candidates: new_list.append(position) seen_candidates.add(position.ballot_item_name) - # TODO: If we have to change all the exising VoterGuidePossibilityPosition entries to match the current - # query, our design could use improvement + # TODO: If we have to change all the exising Position entries to match the voter_guide_possibility_id + # passed into the query, our design could use improvement if position.voter_guide_possibility_parent_id != voter_guide_possibility_id: # position.voter_guide_possibility_parent_id = voter_guide_possibility_id - logger.log("Updating '" + position.ballot_item_name + "' from parent id " + - position.voter_guide_possibility_parent_id + " to " + - voter_guide_possibility_id) + logger.debug("Updating '" + position.ballot_item_name + "' from parent id " + + str(position.voter_guide_possibility_parent_id) + " to " + str(voter_guide_possibility_id)) # position.save() print('-------') From 74956c40377cb6ce7a336c645cb34e568d4184a8 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Thu, 14 Jul 2022 19:42:47 -0700 Subject: [PATCH 53/98] Experiment 24, exploring copy_expert simplification Suspect that there are problems with these tables in the AWS postgres db: 'organization_organization' 'candidate_candidatecampaign' 'polling_location_pollinglocation' so running them last, and seeing how far we get --- retrieve_tables/controllers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index c227f78d3..066c95c85 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -42,7 +42,7 @@ 'organization_organizationreserveddomain', 'party_party', 'politician_politician', - 'politician_politiciansarenotduplicates', + 'politician_politiciansarenotduplicates', # last one to fully run at experiment 23 'position_positionentered', 'twitter_twitterlinktoorganization', 'voter_guide_voterguidepossibility', From 9055d2743e036fdbf00315c748a7016217c006dd Mon Sep 17 00:00:00 2001 From: dalemcgrew Date: Fri, 15 Jul 2022 09:00:05 -0700 Subject: [PATCH 54/98] Added empty alt tags to images and role="presentation" to tables in all email templates. --- .../data_cleanup_organization_analysis.html | 2 +- ...ta_cleanup_organization_list_analysis.html | 6 ++-- .../data_cleanup_voter_list_analysis.html | 2 +- .../campaignx_friend_has_supported.html | 22 ++++++------ .../email_templates/campaignx_news_item.html | 30 ++++++++-------- .../campaignx_super_share_item.html | 34 +++++++++---------- .../campaignx_supporter_initial_response.html | 26 +++++++------- .../friend_accepted_invitation.html | 30 ++++++++-------- .../email_templates/friend_invitation.html | 32 ++++++++--------- .../email_templates/link_to_sign_in.html | 20 +++++------ .../email_templates/message_to_friend.html | 30 ++++++++-------- .../notice_friend_endorsements.html | 32 ++++++++--------- .../notice_voter_daily_summary.html | 34 +++++++++---------- .../send_ballot_to_friends.html | 14 ++++---- .../email_templates/send_ballot_to_self.html | 14 ++++---- .../email_templates/sign_in_code_email.html | 20 +++++------ .../email_templates/verify_email_address.html | 20 +++++------ templates/header_navigation.html | 2 +- 18 files changed, 185 insertions(+), 185 deletions(-) diff --git a/templates/admin_tools/data_cleanup_organization_analysis.html b/templates/admin_tools/data_cleanup_organization_analysis.html index b56223c9c..057de19b6 100644 --- a/templates/admin_tools/data_cleanup_organization_analysis.html +++ b/templates/admin_tools/data_cleanup_organization_analysis.html @@ -174,7 +174,7 @@

Endorsers with Duplicate (Local) Twitter Info

{{ forloop.counter }} {% if organization.organization_photo_url %} - {% endif %} + {% endif %} {{ organization.organization_name }} {% if organization.organization_twitter_handle %}{{ organization.organization_twitter_handle }}
diff --git a/templates/admin_tools/data_cleanup_organization_list_analysis.html b/templates/admin_tools/data_cleanup_organization_list_analysis.html index 04d6badc5..c920afe2d 100644 --- a/templates/admin_tools/data_cleanup_organization_list_analysis.html +++ b/templates/admin_tools/data_cleanup_organization_list_analysis.html @@ -33,7 +33,7 @@

Endorsers with Unique Twitter Data (Local Data)

{{ forloop.counter }} {% if organization.we_vote_hosted_profile_image_url_medium %} - {% endif %} + {% endif %} {{ organization.organization_name }}
   (See images in new window) @@ -101,7 +101,7 @@

Endorsers with Twitter Collision (Local Data)

{{ forloop.counter }} {% if organization.organization_photo_url %} - {% endif %} + {% endif %} {{ organization.organization_name }} {% if organization.organization_twitter_handle %}{{ organization.organization_twitter_handle }}
@@ -145,7 +145,7 @@

Endorsers with Correctly Linked Twitter Data

{{ forloop.counter }} {% if organization.organization_photo_url %} - {% endif %} + {% endif %} {{ organization.organization_name }} {% if organization.organization_twitter_handle %}{{ organization.organization_twitter_handle }}
diff --git a/templates/admin_tools/data_cleanup_voter_list_analysis.html b/templates/admin_tools/data_cleanup_voter_list_analysis.html index 06eee4fc8..1f2952b4f 100644 --- a/templates/admin_tools/data_cleanup_voter_list_analysis.html +++ b/templates/admin_tools/data_cleanup_voter_list_analysis.html @@ -41,7 +41,7 @@

With twitter_id, twitter_screen_name, facebook_id or email

{{ forloop.counter }} {{ voter.id }} - {% if voter.we_vote_hosted_profile_image_url_medium %}{% endif %} + {% if voter.we_vote_hosted_profile_image_url_medium %}{% endif %} {{ voter.get_full_name|default_if_none:"" }} diff --git a/templates/email_outbound/email_templates/campaignx_friend_has_supported.html b/templates/email_outbound/email_templates/campaignx_friend_has_supported.html index 5d4d3f41c..35c3fe955 100644 --- a/templates/email_outbound/email_templates/campaignx_friend_has_supported.html +++ b/templates/email_outbound/email_templates/campaignx_friend_has_supported.html @@ -1,27 +1,27 @@ {# email_outbound/email_templates/campaignx_friend_has_supported.html #} -{# Later we replace "Your friend" with the sender name. #} +{# Later we replace "Your friend" with the sender name. #} We Vote - +