diff --git a/.gitignore b/.gitignore index 5ad200ba6..9eff0d2ae 100644 --- a/.gitignore +++ b/.gitignore @@ -144,6 +144,7 @@ node_modules/ /**/migrations/*.py !/**/migrations/__init__.py config/environment_variables.json +config/environment_variables*.bak.json web_app/build/* # Heroku Related # diff --git a/activity/controllers.py b/activity/controllers.py index 3f1fc06ef..db885efec 100644 --- a/activity/controllers.py +++ b/activity/controllers.py @@ -621,6 +621,28 @@ def notice_friend_endorsements_send( # subject += " is getting ready to vote" activity_description += "is reviewing the ballot" + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + + # Unsubscribe link in email + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/friendopinionsall" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendopinionsall/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_FRIEND_OPINIONS_OTHER_REGIONS_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='friendopinionsall') + # Variables used by templates/email_outbound/email_templates/notice_friend_endorsements.txt and .html template_variables_for_json = { "activity_description": activity_description, @@ -632,10 +654,8 @@ def notice_friend_endorsements_send( "sender_description": speaker_voter_description, "sender_network_details": speaker_voter_network_details, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_email, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", "view_new_endorsements_url": web_app_root_url_verified + "/news/a/" + activity_tidbit_we_vote_id, "view_your_ballot_url": web_app_root_url_verified + "/ballot", } @@ -651,7 +671,10 @@ def notice_friend_endorsements_send( recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " success = outbound_results['success'] if outbound_results['email_outbound_description_saved']: @@ -894,22 +917,41 @@ def notice_voter_daily_summary_send( # NOTICE_VOTER_DAILY_SUMMARY if not positive_value_exists(subject): subject = "Your friends have commented" + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/dailyfriendactivity" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/dailyfriendactivity/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='dailyfriendactivity') + template_variables_for_json = { - "introduction_line": introduction_line, - "subject": subject, - "friend_activity_dict_list": friend_activity_dict_list_modified, + "introduction_line": introduction_line, + "subject": subject, + "friend_activity_dict_list": friend_activity_dict_list_modified, # "sender_name": speaker_voter_name, # "sender_photo": speaker_voter_photo, # "sender_email_address": speaker_voter_email, # Does not affect the "From" email header # "sender_description": speaker_voter_description, # "sender_network_details": speaker_voter_network_details, - "recipient_name": recipient_name, - "recipient_voter_email": recipient_email, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", + "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, + "recipient_voter_email": recipient_email, "view_main_discussion_page_url": web_app_root_url_verified + "/news", - "view_your_ballot_url": web_app_root_url_verified + "/ballot", + "view_your_ballot_url": web_app_root_url_verified + "/ballot", } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) from_email_for_daily_summary = "We Vote " # TODO DALE Make system variable @@ -924,7 +966,10 @@ def notice_voter_daily_summary_send( # NOTICE_VOTER_DAILY_SUMMARY recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " success = outbound_results['success'] if outbound_results['email_outbound_description_saved']: diff --git a/admin_tools/views.py b/admin_tools/views.py index 01d60bf9c..794f51c72 100644 --- a/admin_tools/views.py +++ b/admin_tools/views.py @@ -2,8 +2,8 @@ # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- -from config.base import get_environment_variable, get_git_merge_date, get_node_version, get_postgres_version, \ - get_python_version, LOGIN_URL +from config.base import get_environment_variable, get_node_version, get_postgres_version, \ + get_python_version, LOGIN_URL, get_git_commit_hash from ballot.models import BallotReturned, VoterBallotSaved from candidate.models import CandidateCampaign, CandidateManager from candidate.controllers import candidates_import_from_sample_file @@ -113,7 +113,8 @@ def admin_home_view(request): 'google_civic_election_id': google_civic_election_id, 'python_version': get_python_version(), 'node_version': get_node_version(), - 'git_merge_date': get_git_merge_date(), + 'git_commit_hash': get_git_commit_hash(False), + 'git_commit_hash_url': get_git_commit_hash(True), 'postgres_version': get_postgres_version(), 'shared_link_clicked_unique_sharer_count': shared_link_clicked_unique_sharer_count, 'shared_link_clicked_unique_viewer_count': shared_link_clicked_unique_viewer_count, @@ -1711,7 +1712,7 @@ def login_we_vote(request): voter_manager = VoterManager() voter_device_link_manager = VoterDeviceLinkManager() - results = voter_manager.retrieve_voter_from_voter_device_id(voter_api_device_id) + results = voter_manager.retrieve_voter_from_voter_device_id(voter_api_device_id, read_only=True) if results['voter_found']: voter_on_stage = results['voter'] voter_on_stage_id = voter_on_stage.id diff --git a/analytics/models.py b/analytics/models.py index 9c0b2f775..93f116107 100644 --- a/analytics/models.py +++ b/analytics/models.py @@ -729,7 +729,8 @@ def retrieve_analytics_processed_list( success = False try: - retrieved_voter_we_vote_id_list = voter_list_query.values_list('voter_we_vote_id', flat=True).distinct() + voter_list_query = voter_list_query.values_list('voter_we_vote_id', flat=True).distinct() + retrieved_voter_we_vote_id_list = list(voter_list_query) retrieved_voter_we_vote_id_list_found = True except Exception as e: retrieved_voter_we_vote_id_list_found = False diff --git a/apis_v1/documentation_source/friend_list_doc.py b/apis_v1/documentation_source/friend_list_doc.py index c9d82791b..47dd4b74e 100644 --- a/apis_v1/documentation_source/friend_list_doc.py +++ b/apis_v1/documentation_source/friend_list_doc.py @@ -59,7 +59,7 @@ def friend_list_doc_template_values(url_root): ' "state_code": string,\n' \ ' "kind_of_list": string, \n' \ ' "friend_list": list\n' \ - ' [\n' \ + ' [{\n' \ ' "voter_we_vote_id": string,\n' \ ' "voter_display_name": string,\n' \ ' "voter_photo_url_large": string,\n' \ @@ -74,8 +74,14 @@ def friend_list_doc_template_values(url_root): ' "invitation_status": string,\n' \ ' "invitation_sent_to": string,\n' \ ' "positions_taken": number,\n' \ + ' "mutual_friend_count": number,\n' \ + ' "mutual_friend_preview_list": list\n' \ + ' [{\n' \ + ' "friend_display_name": string,\n' \ + ' "friend_photo_url_medium": string,\n' \ + ' }],\n' \ ' "mutual_friends": number,\n' \ - ' ],\n' \ + ' }],\n' \ '}' template_values = { diff --git a/apis_v1/documentation_source/friend_lists_all_doc.py b/apis_v1/documentation_source/friend_lists_all_doc.py index 746bf1c74..cb2f9da32 100644 --- a/apis_v1/documentation_source/friend_lists_all_doc.py +++ b/apis_v1/documentation_source/friend_lists_all_doc.py @@ -57,7 +57,8 @@ def friend_lists_all_doc_template_values(url_root): 'api_slug': 'friendListsAll', 'api_introduction': "Request information about a voter's friends, including invitations to become a friend, " - "a list of current friends, and friends you share in common with another voter. This API differs from friendList in that it " + "a list of current friends, and friends you share in common with another voter. " + "This API differs from friendList in that it " "returns six different lists at once.", 'try_now_link': 'apis_v1:friendListsAllView', 'url_root': url_root, diff --git a/apis_v1/documentation_source/voter_contact_list_retrieve_doc.py b/apis_v1/documentation_source/voter_contact_list_retrieve_doc.py index 7a3ee4c4e..47e615ba4 100644 --- a/apis_v1/documentation_source/voter_contact_list_retrieve_doc.py +++ b/apis_v1/documentation_source/voter_contact_list_retrieve_doc.py @@ -8,6 +8,11 @@ def voter_contact_list_retrieve_doc_template_values(url_root): Show documentation about voterContactListRetrieve """ required_query_parameter_list = [ + { + 'name': 'voter_device_id', + 'value': 'string', # boolean, integer, long, string + 'description': 'An 88 character unique identifier linked to a voter record on the server', + }, { 'name': 'api_key', 'value': 'string (from post, cookie, or get (in that order))', # boolean, integer, long, string @@ -53,16 +58,17 @@ def voter_contact_list_retrieve_doc_template_values(url_root): ' "has_data_from_google_people_api": boolean,\n' \ ' "ignore_contact": boolean,\n' \ ' "imported_by_voter_we_vote_id": string,\n' \ + ' "is_friend": boolean,\n' \ ' "state_code": string,\n' \ ' },],\n' \ '}' template_values = { - 'api_name': 'voterPlanListRetrieve', - 'api_slug': 'voterPlanListRetrieve', + 'api_name': 'voterContactListRetrieve', + 'api_slug': 'voterContactListRetrieve', 'api_introduction': "Retrieve a voter_contact_list that we can display publicly.", - 'try_now_link': 'apis_v1:voterPlanListRetrieveView', + 'try_now_link': 'apis_v1:voterContactListRetrieveView', 'try_now_link_variables_dict': try_now_link_variables_dict, 'url_root': url_root, 'get_or_post': 'GET', diff --git a/apis_v1/documentation_source/voter_contact_save_doc.py b/apis_v1/documentation_source/voter_contact_save_doc.py new file mode 100644 index 000000000..94fe5e81e --- /dev/null +++ b/apis_v1/documentation_source/voter_contact_save_doc.py @@ -0,0 +1,74 @@ +# apis_v1/documentation_source/voter_contact_save_doc.py +# Brought to you by We Vote. Be good. +# -*- coding: UTF-8 -*- + + +def doc_template_values(url_root): + """ + Show documentation about voterContactSave + """ + required_query_parameter_list = [ + { + 'name': 'api_key', + 'value': 'string (from post, cookie, or get (in that order))', # boolean, integer, long, string + 'description': 'The unique key provided to any organization using the WeVoteServer APIs', + }, + { + 'name': 'voter_device_id', + 'value': 'string', # boolean, integer, long, string + 'description': 'An 88 character unique identifier linked to a voter record on the server. ', + }, + { + 'name': 'email_address_text', + 'value': 'string', # boolean, integer, long, string + 'description': 'The email of the VoterContactEmail that we want to alter.', + }, + ] + + optional_query_parameter_list = [ + { + 'name': 'ignore_voter_contact', + 'value': 'boolean', # boolean, integer, long, string + 'description': 'Set to true if the voter wants to ignore this VoterContact entry.', + }, + ] + + potential_status_codes_list = [ + { + 'code': 'VALID_VOTER_DEVICE_ID_MISSING', + 'description': 'A valid voter_device_id parameter was not included. Cannot proceed.', + }, + { + 'code': 'VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID', + 'description': 'No voter could be found from the voter_device_id', + }, + ] + + try_now_link_variables_dict = { + # 'voter_device_id': '', + } + + api_response = '{\n' \ + ' "success": boolean,\n' \ + ' "status": string,\n' \ + ' "ignore_voter_contact" boolean, \n' \ + ' "email_address_text": string, \n' \ + '}' + + template_values = { + 'api_name': 'voterContactSave', + 'api_slug': 'voterContactSave', + 'api_introduction': + 'Make changes to specific voterContactEmail entries.', + 'try_now_link': 'apis_v1:voterContactSaveView', + 'try_now_link_variables_dict': try_now_link_variables_dict, + 'url_root': url_root, + 'get_or_post': 'get', + 'required_query_parameter_list': required_query_parameter_list, + 'optional_query_parameter_list': optional_query_parameter_list, + 'api_response': api_response, + 'api_response_notes': + "", + 'potential_status_codes_list': potential_status_codes_list, + } + return template_values diff --git a/apis_v1/documentation_source/voter_email_address_save_doc.py b/apis_v1/documentation_source/voter_email_address_save_doc.py index 2199d5f59..a8799762f 100644 --- a/apis_v1/documentation_source/voter_email_address_save_doc.py +++ b/apis_v1/documentation_source/voter_email_address_save_doc.py @@ -80,12 +80,16 @@ def voter_email_address_save_doc_template_values(url_root): ' "text_for_email_address": string,\n' \ ' "make_primary_email": boolean,\n' \ ' "delete_email": boolean,\n' \ - ' "email_address_saved_we_vote_id": string,\n' \ + ' "email_address_we_vote_id": string,\n' \ + ' "email_address_saved_we_vote_id": boolean,\n' \ + ' "email_address_already_owned_by_other_voter": boolean,\n' \ + ' "email_address_already_owned_by_this_voter": boolean,\n' \ ' "email_address_created": boolean,\n' \ - ' "email_address_deleted": boolean,\n' \ ' "email_address_not_valid": boolean,\n' \ + ' "email_address_deleted": boolean,\n' \ ' "verification_email_sent": boolean,\n' \ - ' "email_address_already_owned_by_other_voter": boolean,\n' \ + ' "link_to_sign_in_email_sent": boolean,\n' \ + ' "sign_in_code_email_sent": boolean,\n' \ ' "email_address_found": boolean,\n' \ ' "email_address_list_found": boolean,\n' \ ' "email_address_list": list\n' \ @@ -96,6 +100,7 @@ def voter_email_address_save_doc_template_values(url_root): ' "email_ownership_is_verified": boolean,\n' \ ' "voter_we_vote_id": string,\n' \ ' "email_we_vote_id": string,\n' \ + ' "secret_code_system_locked_for_this_voter_device_id": boolean\n' \ ' ],\n' \ '}' diff --git a/apis_v1/documentation_source/voter_email_address_sign_in_doc.py b/apis_v1/documentation_source/voter_email_address_sign_in_doc.py index 2504b6078..ee5cf8ca4 100644 --- a/apis_v1/documentation_source/voter_email_address_sign_in_doc.py +++ b/apis_v1/documentation_source/voter_email_address_sign_in_doc.py @@ -48,7 +48,10 @@ def voter_email_address_sign_in_doc_template_values(url_root): ' "voter_device_id": string (88 characters long),\n' \ ' "email_ownership_is_verified": boolean,\n' \ ' "email_secret_key_belongs_to_this_voter": boolean,\n' \ + ' "email_sign_in_attempted": boolean,\n' \ ' "email_address_found": boolean,\n' \ + ' "yes_please_merge_accounts": boolean,\n' \ + ' "voter_we_vote_id_from_secret_key": string,\n' \ '}' template_values = { diff --git a/apis_v1/urls.py b/apis_v1/urls.py index 10db01df2..be5848255 100644 --- a/apis_v1/urls.py +++ b/apis_v1/urls.py @@ -93,10 +93,16 @@ # url(r'^couponSummaryRetrieve', # views_donation.coupon_summary_retrieve_for_api_view, name='couponSummaryRetrieve'), # No doc yet # url(r'^defaultPricing', views_donation.default_pricing_for_api_view, name='defaultPricing'), # No doc yet + # re_path(r'^createNewPlan', views_donation.create_new_plan_for_api_view, + # name='createNewCoupon'), + # re_path(r'^deletePlan', views_donation.delete_plan_for_api_view, + # name='deletePlan'), re_path(r'^deviceIdGenerate/$', views_misc.device_id_generate_view, name='deviceIdGenerateView'), re_path(r'^deviceStoreFirebaseCloudMessagingToken/$', views_misc.device_store_firebase_fcm_token_view, name='deviceStoreFirebaseCloudMessagingToken'), + re_path(r'^doesOrgHavePaidPlan', views_donation.does_paid_subscription_exist_for_api, + name='doesOrgHavePaidPlan'), re_path(r'^donationCancelSubscription', views_donation.donation_cancel_subscription_view, name='donationCancelSubscription'), re_path(r'^donationHistory', views_donation.donation_history_list_view, name='donationHistory'), @@ -275,12 +281,8 @@ views_twitter.twitter_retrieve_ids_i_follow_view, name='twitterRetrieveIdsIFollowView'), # re_path(r'^validateCoupon', views_donation.validate_coupon_for_api_view, # name='validateCoupon'), - # re_path(r'^createNewPlan', views_donation.create_new_plan_for_api_view, - # name='createNewCoupon'), - # re_path(r'^deletePlan', views_donation.delete_plan_for_api_view, - # name='deletePlan'), - re_path(r'^doesOrgHavePaidPlan', views_donation.does_paid_subscription_exist_for_api, - name='doesOrgHavePaidPlan'), + re_path(r'^unsubscribeInstant/(?P[:a-zA-Z0-9-_.]+)/(?P[:a-zA-Z0-9-_.]+)/', + views_voter.unsubscribe_instant_view, name='unsubscribe_instant'), re_path(r'^voterAddressRetrieve/', views_voter.voter_address_retrieve_view, name='voterAddressRetrieveView'), re_path(r'^voterAddressSave/', views_voter.voter_address_save_view, name='voterAddressSaveView'), @@ -395,6 +397,8 @@ views_voter.voter_contact_list_retrieve_view, name='voterContactListRetrieveView'), re_path(r'^voterContactListSave/', views_voter.voter_contact_list_save_view, name='voterContactListSaveView'), + re_path(r'^voterContactSave/', views_voter.voter_contact_save_view, + name='voterContactSaveView'), re_path(r'^voterSMSPhoneNumberRetrieve/', views_voter.voter_sms_phone_number_retrieve_view, name='voterSMSPhoneNumberRetrieveView'), re_path(r'^voterSMSPhoneNumberSave/', views_voter.voter_sms_phone_number_save_view, @@ -755,6 +759,8 @@ name='voterContactListRetrieveDocs'), re_path(r'^docs/voterContactListSave/$', views_docs.voter_contact_list_save_doc_view, name='voterContactListSaveDocs'), + re_path(r'^docs/voterContactSave/$', views_docs.voter_contact_save_doc_view, + name='voterContactSaveDocs'), re_path(r'^docs/voterSMSPhoneNumberRetrieve/$', views_docs.voter_sms_phone_number_retrieve_doc_view, name='voterSMSPhoneNumberRetrieveDocs'), re_path(r'^docs/voterSMSPhoneNumberSave/$', diff --git a/apis_v1/views/views_apple.py b/apis_v1/views/views_apple.py index a24d75613..6fc3c3dde 100644 --- a/apis_v1/views/views_apple.py +++ b/apis_v1/views/views_apple.py @@ -18,6 +18,7 @@ logger = wevote_functions.admin.get_logger(__name__) WE_VOTE_SERVER_ROOT_URL = get_environment_variable("WE_VOTE_SERVER_ROOT_URL") +DEBUG_LOGGING = False def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView @@ -50,8 +51,9 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView if results: user_code = results['subject_registered_claim'] email = results['email'] - # for a splunk trail... - # logger.error("splunkapple Not an error: Sign in with Apple iOS, decrypted jwt: " + user_code + " " + email) + if DEBUG_LOGGING: + logger.error("awsApple Not an error: Sign in with Apple iOS, (no email decrypted jwt: " + user_code + + " " + email) voter_manager = VoterManager() voter_device_link_manager = VoterDeviceLinkManager() @@ -82,7 +84,8 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView 'previously_signed_in_voter_found': previously_signed_in_voter_found, 'previously_signed_in_voter_we_vote_id': previously_signed_in_voter_we_vote_id, } - # logger.error('splunkapple ' + status) + if DEBUG_LOGGING: + logger.error('awsApple (no voter_starting_process_we_vote_id): ' + status) return HttpResponse(json.dumps(json_data), content_type='application/json') results = sign_in_with_apple_for_api( @@ -106,7 +109,8 @@ def sign_in_with_apple_view(request): # appleSignInSave appleSignInSaveView ) status += merge_results['status'] - # logger.error('splunkapple ' + status) + if DEBUG_LOGGING: + logger.error('awsApple (after apple_sign_in_save_merge_if_needed): ' + status) json_data = { 'status': status, @@ -220,6 +224,8 @@ def sign_in_with_apple_for_api( except Exception as e: success_siwa = False status += "ERROR_APPLE_USER_NOT_CREATED_OR_UPDATED: " + str(e) + ' ' + if DEBUG_LOGGING: + logger.error("awsApple: ", status) handle_exception(e, logger=logger, exception_message=status) results = { @@ -249,7 +255,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect print("Method is", request.method) if not request.method == 'POST': - logger.error('splunkapple appleSignInOauthRedirectDestination WRONG Method: ' + request.method) + logger.error('awsApple appleSignInOauthRedirectDestination WRONG Method: ' + request.method) try: access_token = request.POST['id_token'] @@ -258,6 +264,9 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect critical_variable_missing = True try: state_dict = json.loads(request.POST['state']) + if DEBUG_LOGGING: + logger.error('awsApple post params on redirect:' + json.dumps(state_dict)) + voter_device_id = state_dict['voter_device_id'] return_url = state_dict['return_url'] except Exception as e: @@ -266,7 +275,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect # print('id_token renamed as access_token: ', access_token) if critical_variable_missing: - logger.error('splunkapple ' + status) + logger.error('awsApple ' + status) return HttpResponseRedirect('https://WeVote.US/applesigninprocess') if 'user' in request.POST: @@ -314,8 +323,8 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect if results: user_code = results['subject_registered_claim'] email = results['email'] - # for a splunk trail... - # logger.error("splunkapple Not an error: Sign in with Apple WebApp, decrypted jwt", results) + if DEBUG_LOGGING: + logger.error("awsApple Not an error: Sign in with Apple WebApp, decrypted jwt", results) # voter_we_vote_id = fetch_voter_we_vote_id_from_voter_device_link(voter_device_id) voter_manager = VoterManager() @@ -339,7 +348,7 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect if not positive_value_exists(voter_starting_process_we_vote_id): logger.error( - 'splunkapple did not receive a voter_we_vote_id from voter_device_id in sign_in_with_apple_oauth_redirect_view ') + 'awsApple didnt receive a voter_we_vote_id from voter_device_id in sign_in_with_apple_oauth_redirect_view') results = sign_in_with_apple_for_api( user_code=user_code, @@ -361,7 +370,8 @@ def sign_in_with_apple_oauth_redirect_view(request): # appleSignInOauthRedirect voter_starting_process=voter_starting_process, ) status += merge_results['status'] - # logger.error('splunkapple ' + status) + if DEBUG_LOGGING: + logger.error('awsApple ' + status) return HttpResponseRedirect(return_url) diff --git a/apis_v1/views/views_docs.py b/apis_v1/views/views_docs.py index ebdfd3713..c8d6a3c4a 100644 --- a/apis_v1/views/views_docs.py +++ b/apis_v1/views/views_docs.py @@ -53,7 +53,7 @@ voter_ballot_items_retrieve_from_google_civic_doc, voter_ballot_list_retrieve_doc, \ voter_bookmark_off_save_doc, \ voter_bookmark_on_save_doc, voter_bookmark_status_retrieve_doc, \ - voter_contact_list_retrieve_doc, voter_contact_list_save_doc, \ + voter_contact_list_retrieve_doc, voter_contact_list_save_doc, voter_contact_save_doc, \ voter_count_doc, voter_create_doc, voter_email_address_retrieve_doc, voter_email_address_save_doc, \ voter_email_address_sign_in_doc, voter_email_address_verify_doc, voter_facebook_save_to_current_account_doc, \ voter_facebook_sign_in_retrieve_doc, voter_facebook_sign_in_save_doc, \ @@ -1557,6 +1557,16 @@ def voter_contact_list_save_doc_view(request): return render(request, 'apis_v1/api_doc_page.html', template_values) +def voter_contact_save_doc_view(request): + """ + Show documentation about voterContactSave + """ + url_root = WE_VOTE_SERVER_ROOT_URL + template_values = voter_contact_save_doc.doc_template_values(url_root) + template_values['voter_api_device_id'] = get_voter_api_device_id(request) + return render(request, 'apis_v1/api_doc_page.html', template_values) + + def voter_reaction_like_off_save_doc_view(request): """ Show documentation about voterReactionLikeOffSave diff --git a/apis_v1/views/views_donation.py b/apis_v1/views/views_donation.py index ef9fb0fe8..f9a9e6846 100644 --- a/apis_v1/views/views_donation.py +++ b/apis_v1/views/views_donation.py @@ -230,7 +230,7 @@ def donation_history_list_view(request): # donationHistory if positive_value_exists(voter_device_id): voter_manager = VoterManager() - results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if not results['voter_found']: logger.error("donation_history_list received invalid voter_device_id: " + voter_device_id) status += "DONATION_HISTORY_LIST-INVALID_VOTER_DEVICE_ID_PASSED " diff --git a/apis_v1/views/views_friend.py b/apis_v1/views/views_friend.py index 6aa62519d..bbc702e4d 100644 --- a/apis_v1/views/views_friend.py +++ b/apis_v1/views/views_friend.py @@ -3,7 +3,8 @@ # -*- coding: UTF-8 -*- from config.base import get_environment_variable from django.http import HttpResponse -from friend.controllers import friend_invitation_by_email_send_for_api, friend_invitation_by_email_verify_for_api, \ +from friend.controllers import friend_acceptance_email_should_be_sent, \ + friend_invitation_by_email_send_for_api, friend_invitation_by_email_verify_for_api, \ friend_invitation_by_we_vote_id_send_for_api, friend_invite_response_for_api, friend_list_for_api, \ friend_lists_all_for_api, friend_invitation_by_facebook_send_for_api, \ friend_invitation_by_facebook_verify_for_api, friend_invitation_information_for_api, message_to_friend_send_for_api @@ -36,15 +37,22 @@ def friend_invitation_by_email_send_view(request): # friendInvitationByEmailSen invitation_message = request.GET.get('invitation_message', "") sender_email_address = request.GET.get('sender_email_address', "") hostname = request.GET.get('hostname', "") - results = friend_invitation_by_email_send_for_api(voter_device_id, email_address_array, first_name_array, - last_name_array, email_addresses_raw, - invitation_message, sender_email_address, - web_app_root_url=hostname) + results = friend_invitation_by_email_send_for_api( + voter_device_id=voter_device_id, + email_address_array=email_address_array, + first_name_array=first_name_array, + last_name_array=last_name_array, + email_addresses_raw=email_addresses_raw, + invitation_message=invitation_message, + sender_email_address=sender_email_address, + web_app_root_url=hostname) json_data = { 'status': results['status'], 'success': results['success'], 'voter_device_id': voter_device_id, 'error_message_to_show_voter': results['error_message_to_show_voter'], + 'success_message_to_show_voter': results['success_message_to_show_voter'], + 'number_of_messages_sent': results['number_of_messages_sent'], 'sender_voter_email_address_missing': results['sender_voter_email_address_missing'], } return HttpResponse(json.dumps(json_data), content_type='application/json') @@ -57,19 +65,38 @@ def friend_invitation_by_email_verify_view(request): # friendInvitationByEmailV :return: """ voter_device_id = get_voter_device_id(request) # We standardize how we take in the voter_device_id + acceptance_email_should_be_sent = positive_value_exists(request.GET.get('acceptance_email_should_be_sent', False)) invitation_secret_key = request.GET.get('invitation_secret_key', "") hostname = request.GET.get('hostname', "") - results = friend_invitation_by_email_verify_for_api(voter_device_id, invitation_secret_key, - web_app_root_url=hostname) - json_data = { - 'status': results['status'], - 'success': results['success'], - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': results['voter_has_data_to_preserve'], - 'invitation_found': results['invitation_found'], - 'attempted_to_approve_own_invitation': results['attempted_to_approve_own_invitation'], - 'invitation_secret_key': invitation_secret_key, - } + if acceptance_email_should_be_sent: + results = friend_acceptance_email_should_be_sent( + voter_device_id, + invitation_secret_key, + web_app_root_url=hostname) + json_data = { + 'status': results['status'], + 'success': results['success'], + 'acceptance_email_should_be_sent': results['acceptance_email_should_be_sent'], + 'attempted_to_approve_own_invitation': results['attempted_to_approve_own_invitation'], + 'invitation_found': results['invitation_found'], + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': results['voter_has_data_to_preserve'], + } + else: + results = friend_invitation_by_email_verify_for_api( + voter_device_id, + invitation_secret_key) + json_data = { + 'status': results['status'], + 'success': results['success'], + 'acceptance_email_should_be_sent': results['acceptance_email_should_be_sent'], + 'attempted_to_approve_own_invitation': results['attempted_to_approve_own_invitation'], + 'invitation_found': results['invitation_found'], + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': results['voter_has_data_to_preserve'], + } return HttpResponse(json.dumps(json_data), content_type='application/json') @@ -128,8 +155,11 @@ def friend_invitation_by_we_vote_id_send_view(request): # friendInvitationByWeV invitation_message = request.GET.get('invitation_message', "") other_voter_we_vote_id = request.GET.get('other_voter_we_vote_id', "") hostname = request.GET.get('hostname', "") - results = friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we_vote_id, invitation_message, - web_app_root_url=hostname) + results = friend_invitation_by_we_vote_id_send_for_api( + voter_device_id=voter_device_id, + other_voter_we_vote_id=other_voter_we_vote_id, + invitation_message=invitation_message, + web_app_root_url=hostname) json_data = { 'status': results['status'], 'success': results['success'], diff --git a/apis_v1/views/views_organization.py b/apis_v1/views/views_organization.py index f8322a1e4..d5164bac8 100644 --- a/apis_v1/views/views_organization.py +++ b/apis_v1/views/views_organization.py @@ -186,7 +186,7 @@ def organization_index_view(request, organization_incoming_domain='', campaign_m chosen_favicon_url_https = None hide_favicon = True else: - # Show the We Vote favicon if a new favicon has not been uploaded and We Vote logo not hidden + # Show We Vote favicon if a new favicon has not been uploaded, and We Vote logo not hidden chosen_favicon_url_https = None hide_favicon = False @@ -201,7 +201,7 @@ def organization_index_view(request, organization_incoming_domain='', campaign_m chosen_social_share_master_image_url_https = None hide_social_share_image = True else: - # Show the We Vote social share image if a new image has not been uploaded and We Vote logo not hidden + # Show We Vote social share image if a new image has not been uploaded, and We Vote logo not hidden chosen_social_share_master_image_url_https = None hide_social_share_image = False @@ -315,7 +315,7 @@ def organization_photos_save_view(request): # organizationPhotosSave voter_has_staff_authority_required = True voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) organization_linked_to_this_voter = False voter_owns_twitter_handle = False if voter_results['voter_found']: @@ -417,9 +417,15 @@ def organization_save_view(request): # organizationSave organization_manager = OrganizationManager() chosen_domain_string = request.GET.get('chosen_domain_string', False) + chosen_domain_string2 = request.GET.get('chosen_domain_string2', False) + chosen_domain_string3 = request.GET.get('chosen_domain_string3', False) # We strip out http or https, and remove paths if positive_value_exists(chosen_domain_string): chosen_domain_string = extract_website_from_url(chosen_domain_string) + if positive_value_exists(chosen_domain_string2): + chosen_domain_string2 = extract_website_from_url(chosen_domain_string2) + if positive_value_exists(chosen_domain_string3): + chosen_domain_string3 = extract_website_from_url(chosen_domain_string3) chosen_subdomain_string = request.GET.get('chosen_subdomain_string', False) chosen_google_analytics_tracking_id = request.GET.get('chosen_google_analytics_tracking_id', False) @@ -438,7 +444,7 @@ def organization_save_view(request): # organizationSave voter_has_staff_authority_required = True voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if voter_results['voter_found']: voter = voter_results['voter'] voter_is_signed_in = voter.is_signed_in() @@ -473,6 +479,8 @@ def organization_save_view(request): # organizationSave 'status': status, 'success': False, 'chosen_domain_string': '', + 'chosen_domain_string2': '', + 'chosen_domain_string3': '', 'full_domain_string_already_taken': None, 'chosen_favicon_url_https': '', 'chosen_google_analytics_tracking_id': '', @@ -522,6 +530,7 @@ def organization_save_view(request): # organizationSave organization_id = results['organization_id'] full_domain_string_already_taken = False full_domain_string_not_valid = False + # Is full domain string valid URL? if positive_value_exists(chosen_domain_string): domain_string_to_test = "https://{chosen_domain_string}" \ "".format(chosen_domain_string=chosen_domain_string) @@ -529,6 +538,19 @@ def organization_save_view(request): # organizationSave full_domain_string_not_valid = True # Do not save it chosen_domain_string = False + if positive_value_exists(chosen_domain_string2): + domain_string_to_test = "https://{chosen_domain_string2}" \ + "".format(chosen_domain_string2=chosen_domain_string2) + if not is_url_valid(domain_string_to_test): + # Do not save it + chosen_domain_string2 = False + if positive_value_exists(chosen_domain_string3): + domain_string_to_test = "https://{chosen_domain_string3}" \ + "".format(chosen_domain_string3=chosen_domain_string3) + if not is_url_valid(domain_string_to_test): + # Do not save it + chosen_domain_string3 = False + # Is full domain string taken by some other group? if positive_value_exists(chosen_domain_string): domain_results = full_domain_string_available( chosen_domain_string, requesting_organization_id=organization_id) @@ -536,6 +558,19 @@ def organization_save_view(request): # organizationSave full_domain_string_already_taken = True # Do not save it chosen_domain_string = False + if positive_value_exists(chosen_domain_string2): + domain_results = full_domain_string_available( + chosen_domain_string2, requesting_organization_id=organization_id) + if not domain_results['full_domain_string_available']: + # Do not save it + chosen_domain_string2 = False + if positive_value_exists(chosen_domain_string3): + domain_results = full_domain_string_available( + chosen_domain_string3, requesting_organization_id=organization_id) + if not domain_results['full_domain_string_available']: + # Do not save it + chosen_domain_string3 = False + subdomain_string_already_taken = False subdomain_string_not_valid = False if positive_value_exists(chosen_subdomain_string): @@ -554,6 +589,8 @@ def organization_save_view(request): # organizationSave chosen_subdomain_string = False else: chosen_domain_string = False + chosen_domain_string2 = False + chosen_domain_string3 = False chosen_subdomain_string = False results = organization_save_for_api( @@ -574,6 +611,8 @@ def organization_save_view(request): # organizationSave facebook_email=facebook_email, facebook_profile_image_url_https=facebook_profile_image_url_https, chosen_domain_string=chosen_domain_string, + chosen_domain_string2=chosen_domain_string2, + chosen_domain_string3=chosen_domain_string3, chosen_google_analytics_tracking_id=chosen_google_analytics_tracking_id, chosen_html_verification_string=chosen_html_verification_string, chosen_hide_we_vote_logo=chosen_hide_we_vote_logo, diff --git a/apis_v1/views/views_voter.py b/apis_v1/views/views_voter.py index 463721b06..a698506bb 100644 --- a/apis_v1/views/views_voter.py +++ b/apis_v1/views/views_voter.py @@ -13,8 +13,9 @@ from django.views.decorators.csrf import csrf_exempt from django.http import HttpResponse from django_user_agents.utils import get_user_agent -from email_outbound.controllers import voter_email_address_save_for_api, voter_email_address_retrieve_for_api, \ - voter_email_address_sign_in_for_api, voter_email_address_verify_for_api +from email_outbound.controllers import voter_email_address_retrieve_for_api, voter_email_address_save_for_api, \ + voter_email_address_send_sign_in_code_email_for_api, voter_email_address_sign_in_for_api, \ + voter_email_address_verify_for_api from email_outbound.models import EmailAddress, EmailManager from wevote_functions.functions import extract_first_name_from_full_name, extract_last_name_from_full_name from follow.controllers import voter_issue_follow_for_api @@ -33,7 +34,8 @@ from sms.models import SMSManager from support_oppose_deciding.controllers import voter_opposing_save, voter_stop_opposing_save, \ voter_stop_supporting_save, voter_supporting_save_for_api -from voter.controllers import voter_address_retrieve_for_api, voter_create_for_api, voter_merge_two_accounts_for_api, \ +from voter.controllers import delete_all_voter_information_permanently, \ + voter_address_retrieve_for_api, voter_create_for_api, voter_merge_two_accounts_for_api, \ voter_merge_two_accounts_action, voter_photo_save_for_api, voter_retrieve_for_api, \ voter_save_photo_from_file_reader, voter_sign_out_for_api, voter_split_into_two_accounts_for_api from voter.controllers_contacts import delete_all_voter_contact_emails_for_voter, save_google_contacts, \ @@ -53,6 +55,81 @@ WE_VOTE_SERVER_ROOT_URL = get_environment_variable("WE_VOTE_SERVER_ROOT_URL") +@csrf_exempt +def unsubscribe_instant_view(request, subscription_secret_key='', unsubscribe_modifier=''): + status = '' + if not positive_value_exists(subscription_secret_key) or not positive_value_exists(unsubscribe_modifier): + status += "MISSING_REQUIRED_UNSUBSCRIBE_VARIABLES " + return HttpResponse('failure ' + status, content_type='text/html') + + email_manager = EmailManager() + voter_manager = VoterManager() + email_results = email_manager.retrieve_email_address_object_from_secret_key( + subscription_secret_key=subscription_secret_key) + if email_results['email_address_object_found']: + status += "UNSUBSCRIBE_INSTANT-EMAIL_ADDRESS_FOUND " + email_address_object = email_results['email_address_object'] + voter_results = voter_manager.retrieve_voter_by_we_vote_id(email_address_object.voter_we_vote_id) + voter_id = voter_results['voter_id'] + else: + status += "EMAIL_ADDRESS_NOT_FOUND " + return HttpResponse('failure ' + status, content_type='text/html') + + notification_flag_integer_to_unset = 0 + from voter.models import NOTIFICATION_FRIEND_MESSAGES_EMAIL, NOTIFICATION_FRIEND_OPINIONS_OTHER_REGIONS_EMAIL, \ + NOTIFICATION_FRIEND_OPINIONS_YOUR_BALLOT_EMAIL, NOTIFICATION_FRIEND_REQUEST_RESPONSES_EMAIL, \ + NOTIFICATION_FRIEND_REQUESTS_EMAIL, NOTIFICATION_LOGIN_EMAIL, NOTIFICATION_NEWSLETTER_OPT_IN, \ + NOTIFICATION_SUGGESTED_FRIENDS_EMAIL, NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL + # NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL = 1024 # When a friend posts something - dailyfriendactivity + # NOTIFICATION_FRIEND_REQUEST_RESPONSES_EMAIL = 4096 # "Show me responses to my friend requests" - friendaccept + # NOTIFICATION_FRIEND_REQUESTS_EMAIL = 2 # "New friend requests" - friendinvite + # NOTIFICATION_FRIEND_MESSAGES_EMAIL = 65536 # "Show me messages from friends" - friendmessage + # NOTIFICATION_FRIEND_OPINIONS_YOUR_BALLOT_EMAIL = 32 # "Friends' opinions (on your ballot)" - friendopinions + # NOTIFICATION_FRIEND_OPINIONS_OTHER_REGIONS_EMAIL = 256 # "Friends' opinions (other regions)" - friendopinionsall + # NOTIFICATION_LOGIN_EMAIL = 16384 # "Show me email login requests" - login + # NOTIFICATION_NEWSLETTER_OPT_IN = 1 # "I would like to receive the We Vote newsletter" - newsletter + # NOTIFICATION_SUGGESTED_FRIENDS_EMAIL = 8 # "Suggestions of people you may know" - suggestedfriend + if unsubscribe_modifier == 'dailyfriendactivity': + notification_flag_integer_to_unset = NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL + elif unsubscribe_modifier == 'friendaccept': + notification_flag_integer_to_unset = NOTIFICATION_FRIEND_REQUEST_RESPONSES_EMAIL + elif unsubscribe_modifier == 'friendinvite': + notification_flag_integer_to_unset = NOTIFICATION_FRIEND_REQUESTS_EMAIL + elif unsubscribe_modifier == 'friendmessage': + notification_flag_integer_to_unset = NOTIFICATION_FRIEND_MESSAGES_EMAIL + elif unsubscribe_modifier == 'friendopinions': + notification_flag_integer_to_unset = NOTIFICATION_FRIEND_OPINIONS_YOUR_BALLOT_EMAIL + elif unsubscribe_modifier == 'friendopinionsall': + notification_flag_integer_to_unset = NOTIFICATION_FRIEND_OPINIONS_OTHER_REGIONS_EMAIL + elif unsubscribe_modifier == 'login': + notification_flag_integer_to_unset = NOTIFICATION_LOGIN_EMAIL + elif unsubscribe_modifier == 'newsletter': + notification_flag_integer_to_unset = NOTIFICATION_NEWSLETTER_OPT_IN + elif unsubscribe_modifier == 'suggestedfriend': + notification_flag_integer_to_unset = NOTIFICATION_SUGGESTED_FRIENDS_EMAIL + + if not positive_value_exists(notification_flag_integer_to_unset): + status += "UNSUBSCRIBE_MODIFIER_NOT_VALID " + return HttpResponse('failure ' + status, content_type='text/html') + + if not positive_value_exists(voter_id): + # TODO This doesn't take into consideration 'friendinvite' which is sent to a person + # before a voter object exists. We need a way to let voter's block messages + # if they aren't users of We Vote + status += "VOTER_ID_NOT_FOUND " + return HttpResponse('failure ' + status, content_type='text/html') + + results = voter_manager.update_voter_by_id( + voter_id, + notification_flag_integer_to_unset=notification_flag_integer_to_unset) + status += results['status'] + success = results['success'] + if not success: + return HttpResponse('failure ' + status, content_type='text/html') + + return HttpResponse('success', content_type='text/html') + + def voter_address_retrieve_view(request): # voterAddressRetrieve """ Retrieve an address for this voter so we can figure out which ballot to display @@ -876,19 +953,51 @@ def voter_create_new_account_view(request): # voterCreateNewAccount is_verified_volunteer = request.GET.get('is_verified_volunteer', False) == 'true' # Check to make sure email isn't attached to existing account in EmailAddress table - email_address_queryset = EmailAddress.objects.all() - email_address_queryset = email_address_queryset.filter( - normalized_email_address__iexact=email, - deleted=False + existing_voter_found = False + email_manager = EmailManager() + voter = None + voter_manager = VoterManager() + existing_email_results = email_manager.retrieve_primary_email_with_ownership_verified( + normalized_email_address=email, ) - email_address_list = list(email_address_queryset) - email_already_in_use = True if len(email_address_list) > 0 else False - if email_already_in_use: - status += "EMAIL_ADDRESS_ALREADY_IN_USE " + if existing_email_results['email_address_object_found']: + email_address_object = existing_email_results['email_address_object'] + voter_we_vote_id = email_address_object.voter_we_vote_id + voter_results = voter_manager.retrieve_voter_by_we_vote_id(voter_we_vote_id) + if voter_results['voter_found']: + voter = voter_results['voter'] + existing_voter_found = True + else: + voter_results = voter_manager.retrieve_voter_by_email(email) + if voter_results['voter_found']: + voter = voter_results['voter'] + existing_voter_found = True + + if existing_voter_found: + voter.set_password(password) + if is_admin: + voter.is_admin = True + if is_analytics_admin: + voter.is_analytics_admin = True + if is_partner_organization: + voter.is_partner_organization = True + if is_political_data_manager: + voter.is_political_data_manager = True + if is_political_data_viewer: + voter.is_political_data_viewer = True + if is_verified_volunteer: + voter.is_verified_volunteer = True + if not positive_value_exists(voter.first_name) and positive_value_exists(first_name): + voter.first_name = first_name + if not positive_value_exists(voter.last_name) and positive_value_exists(last_name): + voter.last_name = last_name + voter.save() + + status += "EMAIL_ADDRESS_ALREADY_IN_USE-UPDATED_VOTER " json_data = { 'status': status, - 'success': False, - 'duplicate_email': True, + 'success': True, + 'duplicate_email': False, 'has_permission': True, } else: @@ -948,19 +1057,26 @@ def voter_email_address_save_view(request): # voterEmailAddressSave is_cordova = positive_value_exists(request.GET.get('is_cordova', False)) hostname = request.GET.get('hostname', '') - results = voter_email_address_save_for_api( - voter_device_id=voter_device_id, - text_for_email_address=text_for_email_address, - incoming_email_we_vote_id=incoming_email_we_vote_id, - send_link_to_sign_in=send_link_to_sign_in, - send_sign_in_code_email=send_sign_in_code_email, - resend_verification_email=resend_verification_email, - resend_verification_code_email=resend_verification_code_email, - make_primary_email=make_primary_email, - delete_email=delete_email, - is_cordova=is_cordova, - web_app_root_url=hostname, - ) + if positive_value_exists(send_sign_in_code_email): + results = voter_email_address_send_sign_in_code_email_for_api( + voter_device_id=voter_device_id, + text_for_email_address=text_for_email_address, + web_app_root_url=hostname, + ) + else: + results = voter_email_address_save_for_api( + voter_device_id=voter_device_id, + text_for_email_address=text_for_email_address, + incoming_email_we_vote_id=incoming_email_we_vote_id, + send_link_to_sign_in=send_link_to_sign_in, + send_sign_in_code_email=send_sign_in_code_email, + resend_verification_email=resend_verification_email, + resend_verification_code_email=resend_verification_code_email, + make_primary_email=make_primary_email, + delete_email=delete_email, + is_cordova=is_cordova, + web_app_root_url=hostname, + ) json_data = { 'status': results['status'], @@ -970,7 +1086,6 @@ def voter_email_address_save_view(request): # voterEmailAddressSave 'make_primary_email': make_primary_email, 'delete_email': delete_email, 'email_address_we_vote_id': results['email_address_we_vote_id'], - 'email_address_saved_we_vote_id': results['email_address_saved_we_vote_id'], 'email_address_already_owned_by_other_voter': results['email_address_already_owned_by_other_voter'], 'email_address_already_owned_by_this_voter': results['email_address_already_owned_by_this_voter'], 'email_address_created': results['email_address_created'], @@ -2061,7 +2176,7 @@ def voter_update_view(request): # voterUpdate try: voter_device_id = get_voter_device_id(request) # We standardize how we take in the voter_device_id except RequestDataTooBig: - status += "RequestDataTooBig" + status += "RequestDataTooBig " json_data = { 'status': status, 'success': False, @@ -2095,6 +2210,7 @@ def voter_update_view(request): # voterUpdate is_post = True if request.method == 'POST' else False if is_post: + delete_voter_account = positive_value_exists(request.POST.get('delete_voter_account', False)) facebook_email_changed = positive_value_exists(request.POST.get('facebook_email_changed', False)) facebook_email = request.POST.get('facebook_email', '') if facebook_email_changed else False facebook_profile_image_url_https_changed = \ @@ -2133,6 +2249,7 @@ def voter_update_view(request): # voterUpdate profile_image_type_currently_active_changed = \ positive_value_exists(request.POST.get('profile_image_type_currently_active_changed', False)) else: + delete_voter_account = positive_value_exists(request.GET.get('delete_voter_account', False)) facebook_email, facebook_email_changed = \ return_string_value_and_changed_boolean_from_get(request, 'facebook_email') facebook_profile_image_url_https, facebook_profile_image_url_https_changed = \ @@ -2225,7 +2342,7 @@ def voter_update_view(request): # voterUpdate else False voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=False) voter_id = voter_results['voter_id'] if not positive_value_exists(voter_id): status += "VOTER_NOT_FOUND_FROM_DEVICE_ID-VOTER_UPDATE " @@ -2260,6 +2377,27 @@ def voter_update_view(request): # voterUpdate # At this point, we have a valid voter voter = voter_results['voter'] voter_we_vote_id = voter.we_vote_id + + if delete_voter_account: + # We want to fully delete this record + results = delete_all_voter_information_permanently(voter_to_delete=voter) + if results['success']: + status += "VOTER_DELETED_COMPLETELY " + json_data = { + 'status': status, + 'success': True, + 'voter_deleted': True, + } + else: + status += "VOTER_NOT_DELETED: " + results['status'] + " " + json_data = { + 'status': status, + 'success': True, + 'voter_not_deleted': True, + } + response = HttpResponse(json.dumps(json_data), content_type='application/json') + return response + voter_full_name_at_start = voter.get_full_name(real_name_only=True) if at_least_one_variable_has_changed or external_voter_id_to_be_saved: @@ -2639,16 +2777,24 @@ def voter_notification_settings_update_view(request): # voterNotificationSettin interface_status_flags = False try: - flag_integer_to_set = request.GET['flag_integer_to_set'] - flag_integer_to_set = flag_integer_to_set.strip() - flag_integer_to_set = convert_to_int(flag_integer_to_set) + flag_integer_to_set_changed = positive_value_exists(request.GET['flag_integer_to_set_changed']) + if flag_integer_to_set_changed: + flag_integer_to_set = request.GET['flag_integer_to_set'] + flag_integer_to_set = flag_integer_to_set.strip() + flag_integer_to_set = convert_to_int(flag_integer_to_set) + else: + flag_integer_to_set = False except KeyError: flag_integer_to_set = False try: - flag_integer_to_unset = request.GET['flag_integer_to_unset'] - flag_integer_to_unset = flag_integer_to_unset.strip() - flag_integer_to_unset = convert_to_int(flag_integer_to_unset) + flag_integer_to_unset_changed = positive_value_exists(request.GET['flag_integer_to_unset_changed']) + if flag_integer_to_unset_changed: + flag_integer_to_unset = request.GET['flag_integer_to_unset'] + flag_integer_to_unset = flag_integer_to_unset.strip() + flag_integer_to_unset = convert_to_int(flag_integer_to_unset) + else: + flag_integer_to_unset = False except KeyError: flag_integer_to_unset = False @@ -2761,7 +2907,7 @@ def voter_notification_settings_update_view(request): # voterNotificationSettin if at_least_one_variable_has_changed: pass else: - # If here, there is nothing to change and we just want to return the latest data from the voter object + # If here, there is nothing to change. We just want to return the latest data from the voter object status += "MISSING_NOTIFICATION_VARIABLE_NO_VARIABLES_PASSED_IN_TO_CHANGE " json_data = { 'status': status, @@ -2938,12 +3084,12 @@ def voter_verify_secret_code_view(request): # voterVerifySecretCode # We get the new email being verified, so we can find the normalized_email_address and check to see # if that is in use by someone else. secret_key_results = email_manager.retrieve_email_address_object_from_secret_key( - voter_device_link.email_secret_key) + email_secret_key=voter_device_link.email_secret_key) if secret_key_results['email_address_object_found']: email_object_from_secret_key = secret_key_results['email_address_object'] matching_results = email_manager.retrieve_email_address_object( - email_object_from_secret_key.normalized_email_address) + normalized_email_address=email_object_from_secret_key.normalized_email_address) if matching_results['email_address_object_found']: email_address_from_normalized = matching_results['email_address_object'] if positive_value_exists(email_address_from_normalized.email_ownership_is_verified): @@ -2982,7 +3128,7 @@ def voter_verify_secret_code_view(request): # voterVerifySecretCode else: # Find and verify the unverified email we are verifying email_results = email_manager.verify_email_address_object_from_secret_key( - voter_device_link.email_secret_key) + email_secret_key=voter_device_link.email_secret_key) if email_results['email_address_object_found']: email_address_object = email_results['email_address_object'] status += "EMAIL_ADDRESS_FOUND_FROM_VERIFY " @@ -3015,6 +3161,7 @@ def voter_verify_secret_code_view(request): # voterVerifySecretCode } return HttpResponse(json.dumps(json_data), content_type='application/json') + @csrf_exempt def voter_contact_list_retrieve_view(request): # voterContactListRetrieve """ @@ -3025,20 +3172,26 @@ def voter_contact_list_retrieve_view(request): # voterContactListRetrieve success = True status = '' status, voter, voter_found, voter_device_link = views_voter_utils.get_voter_from_request(request, status) - voter_contact_email_google_count = 0 + voter_contact_from_apple_api_count = 0 + voter_contact_from_google_api_count = 0 + voter_contact_email_google_count = 0 # Old variable, on way to deprecation + voter_contact_email_list = [] if hasattr(voter, 'we_vote_id'): retrieve_results = voter_contact_list_retrieve_for_api(voter_we_vote_id=voter.we_vote_id) voter_contact_email_list = retrieve_results['voter_contact_email_list'] - voter_contact_email_google_count = retrieve_results['voter_contact_email_google_count'] - + voter_contact_email_google_count = retrieve_results['voter_contact_email_google_count'] # Old variable + voter_contact_from_apple_api_count = retrieve_results['voter_contact_from_apple_api'] + voter_contact_from_google_api_count = retrieve_results['voter_contact_from_google_api'] json_data = { - 'status': status, - 'success': success, - 'voter_contact_email_google_count': voter_contact_email_google_count, - 'voter_contact_email_list': voter_contact_email_list, - 'voter_contact_email_list_count': len(voter_contact_email_list), + 'status': status, + 'success': success, + 'voter_contact_from_apple_api_count': voter_contact_from_apple_api_count, + 'voter_contact_from_google_api_count': voter_contact_from_google_api_count, + 'voter_contact_email_google_count': voter_contact_email_google_count, # Old variable + 'voter_contact_email_list': voter_contact_email_list, + 'voter_contact_email_list_count': len(voter_contact_email_list), } return HttpResponse(json.dumps(json_data), content_type='application/json') @@ -3056,10 +3209,18 @@ def voter_contact_list_save_view(request): # voterContactListSave status, voter, voter_found, voter_device_link = views_voter_utils.get_voter_from_request(request, status) contacts_string = request.POST.get('contacts', None) + + augment_with_location = request.POST.get('augment_voter_contact_emails_with_location', False) + augment_with_location = positive_value_exists(augment_with_location) + + augment_with_we_vote_data = request.POST.get('augment_voter_contact_emails_with_we_vote_data', False) + augment_with_we_vote_data = positive_value_exists(augment_with_we_vote_data) + delete_all_voter_contact_emails = request.POST.get('delete_all_voter_contact_emails', False) + delete_all_voter_contact_emails = positive_value_exists(delete_all_voter_contact_emails) google_api_key_type = request.POST.get('google_api_key_type', 'ballot') - if positive_value_exists(delete_all_voter_contact_emails): + if delete_all_voter_contact_emails: results = delete_all_voter_contact_emails_for_voter(voter_we_vote_id=voter.we_vote_id) elif hasattr(voter, 'we_vote_id') and contacts_string: contacts = json.loads(contacts_string) @@ -3067,11 +3228,13 @@ def voter_contact_list_save_view(request): # voterContactListSave results = save_google_contacts(voter_we_vote_id=voter.we_vote_id, contacts=contacts) status += results['status'] - from email_outbound.controllers import augment_emails_for_voter_with_we_vote_data - augment_results = augment_emails_for_voter_with_we_vote_data(voter_we_vote_id=voter.we_vote_id) - status += augment_results['status'] + if augment_with_we_vote_data: + from email_outbound.controllers import augment_emails_for_voter_with_we_vote_data + augment_results = augment_emails_for_voter_with_we_vote_data(voter_we_vote_id=voter.we_vote_id) + status += augment_results['status'] # 2021-09-30 Requires Pro account which costs $90/month + # 2022-06-06 We are currently paying for this SendGrid account, so we can implement this # from email_outbound.controllers import augment_emails_for_voter_with_sendgrid # augment_results = augment_emails_for_voter_with_sendgrid(voter_we_vote_id=voter.we_vote_id) # status += augment_results['status'] @@ -3081,21 +3244,90 @@ def voter_contact_list_save_view(request): # voterContactListSave # augment_results = augment_emails_for_voter_with_snovio(voter_we_vote_id=voter.we_vote_id) # status += augment_results['status'] - from import_export_open_people.controllers import augment_emails_for_voter_with_open_people - augment_results = augment_emails_for_voter_with_open_people(voter_we_vote_id=voter.we_vote_id) - status += augment_results['status'] + if augment_with_location: + from import_export_open_people.controllers import augment_emails_for_voter_with_open_people + augment_results = augment_emails_for_voter_with_open_people(voter_we_vote_id=voter.we_vote_id) + status += augment_results['status'] retrieve_results = voter_contact_list_retrieve_for_api(voter_we_vote_id=voter.we_vote_id) voter_contact_email_list = retrieve_results['voter_contact_email_list'] voter_contact_email_google_count = retrieve_results['voter_contact_email_google_count'] + augment_sequence_complete = delete_all_voter_contact_emails or augment_with_location + augment_sequence_has_next_step = augment_with_we_vote_data or (contacts_stored > 0) json_data = { 'status': status, 'success': success, - 'we_vote_id_for_google_contacts': voter.we_vote_id, + 'augment_voter_contact_emails_with_location': augment_with_location, + 'augment_voter_contact_emails_with_we_vote_data': augment_with_we_vote_data, 'contacts_stored': contacts_stored, + 'delete_all_voter_contact_emails': delete_all_voter_contact_emails, + 'voter_contact_email_augment_sequence_complete': augment_sequence_complete, + 'voter_contact_email_augment_sequence_has_next_step': augment_sequence_has_next_step, 'voter_contact_email_google_count': voter_contact_email_google_count, 'voter_contact_email_list': voter_contact_email_list, 'voter_contact_email_list_count': len(voter_contact_email_list), + 'we_vote_id_for_google_contacts': voter.we_vote_id, + } + return HttpResponse(json.dumps(json_data), content_type='application/json') + + +@csrf_exempt +def voter_contact_save_view(request): # voterContactSave + success = True + status = '' + email_ignored = False + + status, voter, voter_found, voter_device_link = views_voter_utils.get_voter_from_request(request, status) + email_address_text = request.GET.get('email_address_text', None) + ignore_voter_contact = positive_value_exists(request.GET.get('ignore_voter_contact', False)) + + try: + voter_we_vote_id = voter.we_vote_id + except Exception as e: + status += "VOTER_CONTACT_SAVE_NO_VOTER_WE_VOTE_ID: " + str(e) + " " + voter_we_vote_id = '' + + action_variable_found = positive_value_exists(ignore_voter_contact) + if not positive_value_exists(action_variable_found) or not positive_value_exists(email_address_text) or \ + not positive_value_exists(voter_we_vote_id): + email_ignored = False + status += "VOTER_CONTACT_SAVE_MISSING_KEY_VARIABLE " + success = False + json_data = { + 'status': status, + 'success': success, + 'email_address_text': email_address_text, + 'email_ignored': email_ignored, + } + return HttpResponse(json.dumps(json_data), content_type='application/json') + + voter_manager = VoterManager() + results = voter_manager.retrieve_voter_contact_email( + email_address_text=email_address_text, + imported_by_voter_we_vote_id=voter_we_vote_id, + read_only=False, + ) + if results['voter_contact_email_found']: + voter_contact_email = results['voter_contact_email'] + if ignore_voter_contact: + try: + voter_contact_email.ignore_contact = True + voter_contact_email.save() + email_ignored = True + except Exception as e: + success = False + status += "VOTER_CONTACT_EMAIL_FAILED_TO_SAVE_IGNORE: " + str(e) + ' ' + else: + status += "VOTER_CONTACT_EMAIL_ACTION_MISSING " + else: + status += "VOTER_CONTACT_EMAIL_NOT_FOUND " + json_data = { + 'status': status, + 'success': success, + 'action_variable_found': action_variable_found, + 'email_address_text': email_address_text, + 'email_ignored': email_ignored, + 'ignore_voter_contact': ignore_voter_contact, } return HttpResponse(json.dumps(json_data), content_type='application/json') diff --git a/apis_v1/views/views_voter_guide.py b/apis_v1/views/views_voter_guide.py index 211f957b5..b29ad09b2 100644 --- a/apis_v1/views/views_voter_guide.py +++ b/apis_v1/views/views_voter_guide.py @@ -308,7 +308,7 @@ def voter_guides_from_friends_upcoming_retrieve_view(request): # voterGuidesFro voter_device_id = get_voter_device_id(request) voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if positive_value_exists(voter_results['voter_found']): voter_we_vote_id = voter_results['voter'].we_vote_id else: @@ -322,7 +322,8 @@ def voter_guides_from_friends_upcoming_retrieve_view(request): # voterGuidesFro return HttpResponse(json.dumps(results), content_type='application/json') results = voter_guides_upcoming_retrieve_for_api( - google_civic_election_id_list=google_civic_election_id_list, friends_vs_public=FRIENDS_AND_PUBLIC, + google_civic_election_id_list=google_civic_election_id_list, + friends_vs_public=FRIENDS_AND_PUBLIC, voter_we_vote_id=voter_we_vote_id) status += results['status'] diff --git a/apple/AppleResolver.py b/apple/AppleResolver.py index c635feab2..e749e88cd 100644 --- a/apple/AppleResolver.py +++ b/apple/AppleResolver.py @@ -5,6 +5,7 @@ import wevote_functions.admin logger = wevote_functions.admin.get_logger(__name__) +DEBUG_LOGGING = False class AppleResolver(object): @@ -47,8 +48,9 @@ def authenticate(cls, access_token, client_id): verified_payload = jwt.decode(access_token, apple_public_key_as_string, audience=client_id, - algorithm=public_key_info['alg']) - print('AppleResolver verified_payload: ', verified_payload) + algorithms=["RS256"]) + if DEBUG_LOGGING: + logger.error('awsApple AppleResolver verified_payload: ', verified_payload) # sub: # The subject registered claim identifies the principal that is the subject of the identity token. Since diff --git a/apple/models.py b/apple/models.py index ea684b66f..7e0935ba9 100644 --- a/apple/models.py +++ b/apple/models.py @@ -2,10 +2,7 @@ # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- -from datetime import date from django.db import models -from wevote_functions.functions import positive_value_exists -from exception.models import handle_exception, print_to_log import wevote_functions.admin logger = wevote_functions.admin.get_logger(__name__) @@ -21,6 +18,7 @@ class AppleUser(models.Model): # objects = None # voter_device_id = models.CharField(verbose_name='voter device id', # max_length=255, null=False, blank=False, unique=True, default='DELETE_ME') + objects = None voter_we_vote_id = models.CharField(verbose_name="we vote id for the Apple ID owner", max_length=255, unique=True) user_code = models.CharField(verbose_name="User's apple id code", max_length=255, null=False, unique=False) email = models.EmailField(verbose_name='apple email address', max_length=255, unique=False, @@ -32,7 +30,7 @@ class AppleUser(models.Model): last_name = models.CharField( verbose_name="User's last_name from Apple", max_length=255, null=True, blank=True, unique=False) - # The next three are for debugging/statistics are are not necessary for sign in + # The next three are for debugging/statistics are not necessary for sign in apple_platform = models.CharField( verbose_name="Platform of Apple Device", max_length=32, null=True, blank=True, unique=False) apple_os_version = models.CharField( diff --git a/ballot/controllers.py b/ballot/controllers.py index 2db874b45..cb44ca01d 100644 --- a/ballot/controllers.py +++ b/ballot/controllers.py @@ -1011,28 +1011,28 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve if not positive_value_exists(voter_id): status += " " + "VALID_VOTER_ID_MISSING" error_json_data = { - 'status': status, - 'success': False, - 'ballot_caveat': '', - 'ballot_found': False, - 'ballot_item_list': [], + 'status': status, + 'success': False, + 'ballot_caveat': '', + 'ballot_found': False, + 'ballot_item_list': [], 'ballot_location_display_name': '', 'ballot_location_shortcut': ballot_location_shortcut, 'ballot_returned_we_vote_id': ballot_returned_we_vote_id, - 'google_civic_election_id': google_civic_election_id, - 'is_from_substituted_address': False, - 'is_from_test_ballot': False, + 'google_civic_election_id': google_civic_election_id, + 'is_from_substituted_address': False, + 'is_from_test_ballot': False, 'next_national_election_day_text': '', - 'original_text_city': '', - 'original_text_state': '', - 'original_text_zip': '', + 'original_text_city': '', + 'original_text_state': '', + 'original_text_zip': '', 'polling_location_we_vote_id_source': '', - 'substituted_address_nearby': '', - 'substituted_address_city': '', - 'substituted_address_state': '', - 'substituted_address_zip': '', - 'text_for_map_search': '', - 'voter_device_id': voter_device_id, + 'substituted_address_nearby': '', + 'substituted_address_city': '', + 'substituted_address_state': '', + 'substituted_address_zip': '', + 'text_for_map_search': '', + 'voter_device_id': voter_device_id, } return error_json_data @@ -1043,6 +1043,9 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve status += " " + voter_address_results['status'] # Note that this might be an empty VoterAddress object voter_address = voter_address_results['voter_address'] + text_for_map_search = '' + if voter_address and hasattr(voter_address, 'text_for_map_search'): + text_for_map_search = voter_address.text_for_map_search if positive_value_exists(voter_address_results['voter_address_has_value']): ballot_retrieval_based_on_voter_address = True else: @@ -1062,35 +1065,35 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve elif positive_value_exists(ballot_location_shortcut): ballot_caveat = "We could not find the ballot '{ballot_location_shortcut}'.".format( ballot_location_shortcut=ballot_location_shortcut) - elif positive_value_exists(voter_address.text_for_map_search): + elif positive_value_exists(text_for_map_search): ballot_caveat = "We could not find a ballot near '{text_for_map_search}'.".format( - text_for_map_search=voter_address.text_for_map_search) + text_for_map_search=text_for_map_search) else: ballot_caveat = "Please save your address so we can find your ballot." error_json_data = { - 'status': status, - 'success': True, - 'ballot_caveat': ballot_caveat, - 'ballot_found': False, - 'ballot_item_list': [], + 'status': status, + 'success': True, + 'ballot_caveat': ballot_caveat, + 'ballot_found': False, + 'ballot_item_list': [], 'ballot_location_display_name': '', 'ballot_location_shortcut': ballot_location_shortcut, 'ballot_returned_we_vote_id': ballot_returned_we_vote_id, - 'google_civic_election_id': google_civic_election_id, - 'is_from_substituted_address': False, - 'is_from_test_ballot': False, + 'google_civic_election_id': google_civic_election_id, + 'is_from_substituted_address': False, + 'is_from_test_ballot': False, 'next_national_election_day_text': next_national_election_day_text, - 'original_text_city': '', - 'original_text_state': '', - 'original_text_zip': '', - 'polling_location_we_vote_id_source': '', - 'substituted_address_nearby': '', - 'substituted_address_city': '', - 'substituted_address_state': '', - 'substituted_address_zip': '', - 'text_for_map_search': voter_address.text_for_map_search, - 'voter_device_id': voter_device_id, + 'original_text_city': '', + 'original_text_state': '', + 'original_text_zip': '', + 'polling_location_we_vote_id_source': '', + 'substituted_address_nearby': '', + 'substituted_address_city': '', + 'substituted_address_state': '', + 'substituted_address_zip': '', + 'text_for_map_search': text_for_map_search, + 'voter_device_id': voter_device_id, } return error_json_data @@ -1166,34 +1169,34 @@ def voter_ballot_items_retrieve_for_api( # voterBallotItemsRetrieve status += " " + results['status'] json_data = { - 'status': status, - 'success': True, - 'ballot_caveat': voter_ballot_saved.ballot_caveat(), - 'ballot_found': True, - 'ballot_item_list': results['ballot_item_list'], - 'ballot_location_display_name': voter_ballot_saved.ballot_location_display_name, - 'ballot_location_shortcut': voter_ballot_saved.ballot_location_shortcut, - 'ballot_returned_we_vote_id': voter_ballot_saved.ballot_returned_we_vote_id, - 'election_name': voter_ballot_saved.election_description_text, - 'election_day_text': voter_ballot_saved.election_day_text(), - 'google_civic_election_id': google_civic_election_id, - 'is_from_substituted_address': voter_ballot_saved.is_from_substituted_address, - 'is_from_test_ballot': voter_ballot_saved.is_from_test_ballot, + 'status': status, + 'success': True, + 'ballot_caveat': voter_ballot_saved.ballot_caveat(), + 'ballot_found': True, + 'ballot_item_list': results['ballot_item_list'], + 'ballot_location_display_name': voter_ballot_saved.ballot_location_display_name, + 'ballot_location_shortcut': voter_ballot_saved.ballot_location_shortcut, + 'ballot_returned_we_vote_id': voter_ballot_saved.ballot_returned_we_vote_id, + 'election_name': voter_ballot_saved.election_description_text, + 'election_day_text': voter_ballot_saved.election_day_text(), + 'google_civic_election_id': google_civic_election_id, + 'is_from_substituted_address': voter_ballot_saved.is_from_substituted_address, + 'is_from_test_ballot': voter_ballot_saved.is_from_test_ballot, 'next_national_election_day_text': next_national_election_day_text, - 'original_text_city': voter_ballot_saved.original_text_city, - 'original_text_state': voter_ballot_saved.original_text_state, - 'original_text_zip': voter_ballot_saved.original_text_zip, - 'polling_location_we_vote_id_source': voter_ballot_saved.polling_location_we_vote_id_source, - 'substituted_address_nearby': voter_ballot_saved.substituted_address_nearby, - 'substituted_address_city': voter_ballot_saved.substituted_address_city, - 'substituted_address_state': voter_ballot_saved.substituted_address_state, - 'substituted_address_zip': voter_ballot_saved.substituted_address_zip, - 'text_for_map_search': voter_ballot_saved.original_text_for_map_search, - 'voter_device_id': voter_device_id, + 'original_text_city': voter_ballot_saved.original_text_city, + 'original_text_state': voter_ballot_saved.original_text_state, + 'original_text_zip': voter_ballot_saved.original_text_zip, + 'polling_location_we_vote_id_source': voter_ballot_saved.polling_location_we_vote_id_source, + 'substituted_address_nearby': voter_ballot_saved.substituted_address_nearby, + 'substituted_address_city': voter_ballot_saved.substituted_address_city, + 'substituted_address_state': voter_ballot_saved.substituted_address_state, + 'substituted_address_zip': voter_ballot_saved.substituted_address_zip, + 'text_for_map_search': voter_ballot_saved.original_text_for_map_search, + 'voter_device_id': voter_device_id, } return json_data - status += " " + "NO_VOTER_BALLOT_SAVED_FOUND" + status += " " + "NO_VOTER_BALLOT_SAVED_FOUND " json_data = { 'status': status, 'success': True, @@ -1847,6 +1850,7 @@ def voter_ballot_list_retrieve_for_api(voter_id): # voterBallotListRetrieve "original_text_for_map_search": one_ballot_entry.original_text_for_map_search, "ballot_returned_we_vote_id": ballot_returned_we_vote_id, "ballot_location_shortcut": one_ballot_entry.ballot_location_shortcut, + "state_code": one_ballot_entry.state_code, "state_code_list": state_code_list, } voter_ballot_list_for_json.append(one_voter_ballot_list) @@ -1875,6 +1879,7 @@ def voter_ballot_list_retrieve_for_api(voter_id): # voterBallotListRetrieve "original_text_for_map_search": "", "ballot_returned_we_vote_id": "", "ballot_location_shortcut": "", + "state_code": election.state_code, "state_code_list": state_code_list, } final_ballot_list.append(one_election) @@ -2675,6 +2680,25 @@ def voter_ballot_items_retrieve_for_one_election_for_api( success = False if success: + # Loop through measures to make sure we have full measure data needed + contest_measure_we_vote_id_list = [] + for ballot_item in ballot_item_list: + if ballot_item.contest_measure_we_vote_id and \ + ballot_item.contest_measure_we_vote_id not in contest_measure_we_vote_id_list: + contest_measure_we_vote_id_list.append(ballot_item.contest_measure_we_vote_id) + + measure_results_dict = {} + if len(contest_measure_we_vote_id_list) > 0: + # Retrieve all of these measures with a single call + measure_list_manager = ContestMeasureListManager() + results = measure_list_manager.retrieve_measures( + measure_we_vote_id_list=contest_measure_we_vote_id_list, read_only=True) + if results['measure_list_found']: + measure_list_objects = results['measure_list_objects'] + for one_measure in measure_list_objects: + measure_results_dict[one_measure.we_vote_id] = one_measure + + # Now prepare the full list for json result status += "BALLOT_ITEM_LIST_FOUND " for ballot_item in ballot_item_list: if ballot_item.contest_office_we_vote_id: @@ -2775,23 +2799,45 @@ def voter_ballot_items_retrieve_for_one_election_for_api( measure_id = ballot_item.contest_measure_id measure_we_vote_id = ballot_item.contest_measure_we_vote_id measure_display_name_number = 100 + try: + if measure_we_vote_id in measure_results_dict: + ballot_item_display_name = measure_results_dict[measure_we_vote_id].measure_title + measure_subtitle = measure_results_dict[measure_we_vote_id].measure_subtitle + measure_text = measure_results_dict[measure_we_vote_id].measure_text + measure_url = measure_results_dict[measure_we_vote_id].measure_url + no_vote_description = measure_results_dict[measure_we_vote_id].ballotpedia_no_vote_description + yes_vote_description = measure_results_dict[measure_we_vote_id].ballotpedia_yes_vote_description + else: + ballot_item_display_name = ballot_item.ballot_item_display_name + measure_subtitle = ballot_item.measure_subtitle + measure_text = ballot_item.measure_text + measure_url = ballot_item.measure_url + no_vote_description = ballot_item.no_vote_description + yes_vote_description = ballot_item.yes_vote_description + except Exception as e: + ballot_item_display_name = ballot_item.ballot_item_display_name + measure_subtitle = ballot_item.measure_subtitle + measure_text = ballot_item.measure_text + measure_url = ballot_item.measure_url + no_vote_description = ballot_item.no_vote_description + yes_vote_description = ballot_item.yes_vote_description one_ballot_item = { - 'ballot_item_display_name': ballot_item.ballot_item_display_name, + 'ballot_item_display_name': ballot_item_display_name, 'google_civic_election_id': google_civic_election_id, 'google_ballot_placement': ballot_item.google_ballot_placement, 'id': measure_id, 'kind_of_ballot_item': kind_of_ballot_item, 'local_ballot_order': ballot_item.local_ballot_order + 100, # Shift to bottom - 'measure_subtitle': ballot_item.measure_subtitle, - 'measure_text': ballot_item.measure_text, - 'measure_url': ballot_item.measure_url, - 'no_vote_description': strip_html_tags(ballot_item.no_vote_description), + 'measure_subtitle': measure_subtitle, + 'measure_text': measure_text, + 'measure_url': measure_url, + 'no_vote_description': strip_html_tags(no_vote_description), 'district_name': "", # TODO Add this 'election_display_name': "", # TODO Add this 'regional_display_name': "", # TODO Add this 'state_display_name': "", # TODO Add this 'we_vote_id': measure_we_vote_id, - 'yes_vote_description': strip_html_tags(ballot_item.yes_vote_description), + 'yes_vote_description': strip_html_tags(yes_vote_description), } ballot_items_to_display.append(one_ballot_item.copy()) diff --git a/ballot/models.py b/ballot/models.py index 9dda2c3e4..ab6a57857 100644 --- a/ballot/models.py +++ b/ballot/models.py @@ -1033,7 +1033,7 @@ def count_ballot_items(self, google_civic_election_id, state_code=""): success = False status = '' try: - ballot_item_queryset = BallotItem.objects.all() + ballot_item_queryset = BallotItem.objects.using('readonly').all() ballot_item_queryset = ballot_item_queryset.filter(google_civic_election_id=google_civic_election_id) if positive_value_exists(state_code): ballot_item_queryset = ballot_item_queryset.filter(state_code__iexact=state_code) @@ -1063,7 +1063,7 @@ def count_ballot_items_for_election_lacking_state(self, google_civic_election_id success = False status = '' try: - ballot_item_queryset = BallotItem.objects.all() + ballot_item_queryset = BallotItem.objects.using('readonly').all() ballot_item_queryset = ballot_item_queryset.filter(google_civic_election_id=google_civic_election_id) ballot_item_queryset = ballot_item_queryset.filter(Q(state_code=None) | Q(state_code="")) ballot_item_list_count = ballot_item_queryset.count() @@ -1395,7 +1395,7 @@ def fetch_most_recent_google_civic_election_id(self): if results['success']: election_list = results['election_list'] for one_election in election_list: - ballot_item_queryset = BallotItem.objects.all() + ballot_item_queryset = BallotItem.objects.using('readonly').all() ballot_item_queryset = ballot_item_queryset.filter( google_civic_election_id=one_election.google_civic_election_id) number_found = ballot_item_queryset.count() @@ -1410,7 +1410,7 @@ def fetch_ballot_item_list_count_for_ballot_returned(self, voter_id, polling_loc voter_id = convert_to_int(voter_id) google_civic_election_id = convert_to_int(google_civic_election_id) try: - ballot_item_queryset = BallotItem.objects.all() + ballot_item_queryset = BallotItem.objects.using('readonly').all() if positive_value_exists(voter_id): ballot_item_queryset = ballot_item_queryset.filter( voter_id=voter_id) @@ -3057,7 +3057,7 @@ class BallotReturnedListManager(models.Manager): def fetch_ballot_location_display_option_on_count_for_election(self, google_civic_election_id, state_code=''): google_civic_election_id = convert_to_int(google_civic_election_id) try: - ballot_returned_queryset = BallotReturned.objects.all() + ballot_returned_queryset = BallotReturned.objects.using('readonly').all() ballot_returned_queryset = ballot_returned_queryset.filter( google_civic_election_id=google_civic_election_id) ballot_returned_queryset = ballot_returned_queryset.filter(ballot_location_display_option_on=True) diff --git a/ballot/views_admin.py b/ballot/views_admin.py index 7cf31be6f..6fbab4a82 100644 --- a/ballot/views_admin.py +++ b/ballot/views_admin.py @@ -48,7 +48,7 @@ def ballot_items_sync_out_view(request): # ballotItemsSyncOut return HttpResponse(json.dumps(json_data), content_type='application/json') try: - ballot_item_list = BallotItem.objects.all() + ballot_item_list = BallotItem.objects.using('readonly').all() # We only want BallotItem values associated with map points ballot_item_list = ballot_item_list.exclude( Q(polling_location_we_vote_id__isnull=True) | Q(polling_location_we_vote_id="")) diff --git a/campaign/controllers.py b/campaign/controllers.py index 7e6d482f0..ec9cc0825 100644 --- a/campaign/controllers.py +++ b/campaign/controllers.py @@ -342,7 +342,7 @@ def campaignx_news_item_save_for_api( # campaignNewsItemSave success = True voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if voter_results['voter_found']: voter = voter_results['voter'] voter_we_vote_id = voter.we_vote_id diff --git a/campaign/controllers_email_outbound.py b/campaign/controllers_email_outbound.py index 13fbefceb..98aba29ba 100644 --- a/campaign/controllers_email_outbound.py +++ b/campaign/controllers_email_outbound.py @@ -118,6 +118,27 @@ def campaignx_friend_has_supported_send( # CAMPAIGNX_FRIEND_HAS_SUPPORTED_TEMPL subject = your_friends_name + " supports " + campaignx_title politician_full_sentence_string = '' + # Unsubscribe link in email + recipient_unsubscribe_url = \ + campaigns_root_url_verified + "/settings/notifications/esk/" + recipient_email_subscription_secret_key + # recipient_unsubscribe_url = \ + # "{root_url}/unsubscribe/{email_secret_key}/friendcampaignsupport" \ + # "".format( + # email_secret_key=recipient_email_subscription_secret_key, + # root_url=campaigns_root_url_verified, + # ) + # Instant unsubscribe link in email header + # list_unsubscribe_url = \ + # "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendcampaignsupport/" \ + # "".format( + # email_secret_key=recipient_email_subscription_secret_key, + # root_url=WE_VOTE_SERVER_ROOT_URL, + # ) + # # Instant unsubscribe email address in email header + # # from voter.models import NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL # To be updated + # list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + # "".format(setting='friendcampaignsupport') + template_variables_for_json = { "subject": subject, "campaignx_title": campaignx_title, @@ -125,11 +146,9 @@ def campaignx_friend_has_supported_send( # CAMPAIGNX_FRIEND_HAS_SUPPORTED_TEMPL "politician_count": politician_count, "politician_full_sentence_string": politician_full_sentence_string, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_email, - "recipient_unsubscribe_url": campaigns_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, "speaker_voter_name": speaker_voter_name, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", "view_main_discussion_page_url": campaigns_root_url_verified + "/news", "view_your_ballot_url": campaigns_root_url_verified + "/ballot", "we_vote_hosted_campaign_photo_large_url": we_vote_hosted_campaign_photo_large_url, @@ -147,7 +166,10 @@ def campaignx_friend_has_supported_send( # CAMPAIGNX_FRIEND_HAS_SUPPORTED_TEMPL recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + # list_unsubscribe_mailto=list_unsubscribe_mailto, + # list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " success = outbound_results['success'] if outbound_results['email_outbound_description_saved']: @@ -262,6 +284,27 @@ def campaignx_news_item_send( # CAMPAIGNX_NEWS_ITEM_TEMPLATE campaignx_news_item_url = campaignx_url + '/u/' + campaignx_news_item_we_vote_id + # Unsubscribe link in email + recipient_unsubscribe_url = \ + campaigns_root_url_verified + "/settings/notifications/esk/" + recipient_email_subscription_secret_key + # recipient_unsubscribe_url = \ + # "{root_url}/unsubscribe/{email_secret_key}/friendopinionsall" \ + # "".format( + # email_secret_key=recipient_email_subscription_secret_key, + # root_url=campaigns_root_url_verified, + # ) + # # Instant unsubscribe link in email header + # list_unsubscribe_url = \ + # "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendopinionsall/" \ + # "".format( + # email_secret_key=recipient_email_subscription_secret_key, + # root_url=WE_VOTE_SERVER_ROOT_URL, + # ) + # # Instant unsubscribe email address in email header + # # from voter.models import NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL # To be updated + # list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + # "".format(setting='friendopinionsall') + template_variables_for_json = { "subject": statement_subject, "campaignx_title": campaignx_title, @@ -272,11 +315,9 @@ def campaignx_news_item_send( # CAMPAIGNX_NEWS_ITEM_TEMPLATE "politician_count": politician_count, "politician_full_sentence_string": politician_full_sentence_string, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_email, - "recipient_unsubscribe_url": campaigns_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, "speaker_voter_name": speaker_voter_name, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", "view_main_discussion_page_url": campaigns_root_url_verified + "/news", "view_your_ballot_url": campaigns_root_url_verified + "/ballot", "we_vote_hosted_campaign_photo_large_url": we_vote_hosted_campaign_photo_large_url, @@ -294,7 +335,10 @@ def campaignx_news_item_send( # CAMPAIGNX_NEWS_ITEM_TEMPLATE recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + # list_unsubscribe_mailto=list_unsubscribe_mailto, + # list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " success = outbound_results['success'] if outbound_results['email_outbound_description_saved']: @@ -355,16 +399,36 @@ def campaignx_super_share_item_send( # CAMPAIGNX_SUPER_SHARE_ITEM_TEMPLATE else: campaignx_news_item_url = '' + recipient_email_subscription_secret_key = '' # To be added + # Unsubscribe link in email + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/campaignshare" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=campaigns_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/campaignshare/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL # To be updated + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='campaignshare') + template_variables_for_json = { "campaignx_title": campaignx_title, "campaignx_news_item_url": campaignx_news_item_url, "campaignx_news_text": statement_text_preview, "campaignx_url": view_shared_campaignx_url, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", "invitation_message": invitation_message, "invitation_message_plain_text": invitation_message_plain_text, "recipient_email_address": recipient_email_address, "recipient_first_name": recipient_first_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "sender_email_address": speaker_email_address, "sender_photo": speaker_photo, "sender_name": speaker_voter_name, @@ -385,7 +449,10 @@ def campaignx_super_share_item_send( # CAMPAIGNX_SUPER_SHARE_ITEM_TEMPLATE recipient_email_we_vote_id='', recipient_voter_email=recipient_email_address, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " success = outbound_results['success'] if outbound_results['email_outbound_description_saved']: @@ -507,6 +574,28 @@ def campaignx_supporter_initial_response_send( # CAMPAIGNX_SUPPORTER_INITIAL_RE subject = "You support " + campaignx_title politician_full_sentence_string = '' + recipient_email_subscription_secret_key = '' # To be added + # Unsubscribe link in email + recipient_unsubscribe_url = \ + campaigns_root_url_verified + "/settings/notifications/esk/" + recipient_email_subscription_secret_key + # recipient_unsubscribe_url = \ + # "{root_url}/unsubscribe/{email_secret_key}/supporterinitial" \ + # "".format( + # email_secret_key=recipient_email_subscription_secret_key, + # root_url=campaigns_root_url_verified, + # ) + # # Instant unsubscribe link in email header + # list_unsubscribe_url = \ + # "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/supporterinitial/" \ + # "".format( + # email_secret_key=recipient_email_subscription_secret_key, + # root_url=WE_VOTE_SERVER_ROOT_URL, + # ) + # # Instant unsubscribe email address in email header + # # from voter.models import NOTIFICATION_VOTER_DAILY_SUMMARY_EMAIL # To be updated + # list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + # "".format(setting='supporterinitial') + template_variables_for_json = { "subject": subject, "campaignx_share_campaign_url": campaignx_share_campaign_url, @@ -518,10 +607,8 @@ def campaignx_supporter_initial_response_send( # CAMPAIGNX_SUPPORTER_INITIAL_RE # "sender_description": speaker_voter_description, # "sender_network_details": speaker_voter_network_details, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_email, - "recipient_unsubscribe_url": campaigns_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", "view_main_discussion_page_url": campaigns_root_url_verified + "/news", "view_your_ballot_url": campaigns_root_url_verified + "/ballot", "we_vote_hosted_campaign_photo_large_url": we_vote_hosted_campaign_photo_large_url, @@ -539,7 +626,10 @@ def campaignx_supporter_initial_response_send( # CAMPAIGNX_SUPPORTER_INITIAL_RE recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + # list_unsubscribe_mailto=list_unsubscribe_mailto, + # list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " success = outbound_results['success'] if outbound_results['email_outbound_description_saved']: diff --git a/candidate/controllers.py b/candidate/controllers.py index ed209b0f7..aca263734 100644 --- a/candidate/controllers.py +++ b/candidate/controllers.py @@ -205,7 +205,7 @@ def fetch_duplicate_candidate_count(we_vote_candidate, ignore_candidate_id_list) ignore_candidate_id_list=ignore_candidate_id_list) -def find_duplicate_candidate(we_vote_candidate, ignore_candidate_id_list): +def find_duplicate_candidate(we_vote_candidate, ignore_candidate_id_list, read_only=True): if not hasattr(we_vote_candidate, 'candidate_name'): error_results = { 'success': False, @@ -229,6 +229,7 @@ def find_duplicate_candidate(we_vote_candidate, ignore_candidate_id_list): ignore_candidate_id_list=ignore_candidate_id_list, state_code=we_vote_candidate.state_code, vote_usa_politician_id=we_vote_candidate.vote_usa_politician_id, + read_only=read_only, ) if results['candidate_found']: @@ -466,7 +467,7 @@ def merge_these_two_candidates(candidate1_we_vote_id, candidate2_we_vote_id, adm # Candidate 1 is the one we keep, and Candidate 2 is the one we will merge into Candidate 1 candidate1_results = \ - candidate_manager.retrieve_candidate_from_we_vote_id(candidate1_we_vote_id) + candidate_manager.retrieve_candidate_from_we_vote_id(candidate1_we_vote_id, read_only=False) if candidate1_results['candidate_found']: candidate1_on_stage = candidate1_results['candidate'] candidate1_id = candidate1_on_stage.id @@ -480,7 +481,7 @@ def merge_these_two_candidates(candidate1_we_vote_id, candidate2_we_vote_id, adm return results candidate2_results = \ - candidate_manager.retrieve_candidate_from_we_vote_id(candidate2_we_vote_id) + candidate_manager.retrieve_candidate_from_we_vote_id(candidate2_we_vote_id, read_only=False) if candidate2_results['candidate_found']: candidate2_on_stage = candidate2_results['candidate'] candidate2_id = candidate2_on_stage.id @@ -808,7 +809,7 @@ def filter_candidates_structured_json_for_local_duplicates(structured_json): candidate_name, google_civic_candidate_name, google_civic_candidate_name2, google_civic_candidate_name3, google_civic_election_id, contest_office_we_vote_id, politician_we_vote_id, candidate_twitter_handle, ballotpedia_candidate_id, vote_smart_id, maplight_id, - we_vote_id_from_master) + we_vote_id_from_master, read_only=True) if results['candidate_list_found']: # print("Skipping candidate " + str(candidate_name) + ", " + str(google_civic_candidate_name) + ", " + @@ -1175,11 +1176,11 @@ def candidate_retrieve_for_api(candidate_id, candidate_we_vote_id): # candidate candidate_manager = CandidateManager() if positive_value_exists(candidate_id): - results = candidate_manager.retrieve_candidate_from_id(candidate_id) + results = candidate_manager.retrieve_candidate_from_id(candidate_id, read_only=True) success = results['success'] status = results['status'] elif positive_value_exists(candidate_we_vote_id): - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) success = results['success'] status = results['status'] else: @@ -1468,7 +1469,7 @@ def refresh_candidate_data_from_master_tables(candidate_we_vote_id): candidate = CandidateCampaign() twitter_user_manager = TwitterUserManager() - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=False) if not results['candidate_found']: status = "REFRESH_CANDIDATE_FROM_MASTER_TABLES-CANDIDATE_NOT_FOUND " results = { @@ -1552,7 +1553,7 @@ def refresh_candidate_data_from_master_tables(candidate_we_vote_id): def push_candidate_data_to_other_table_caches(candidate_we_vote_id): candidate_manager = CandidateManager() - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=False) candidate = results['candidate'] save_position_from_candidate_results = update_all_position_details_from_candidate(candidate) @@ -1802,7 +1803,8 @@ def retrieve_candidate_list_for_entire_year( candidate_year=candidate_year, limit_to_this_state_code=limit_to_this_state_code, search_string=False, - return_list_of_objects=False) + return_list_of_objects=False, + read_only=True) if results['candidate_list_found']: candidate_list_found = True candidate_list_light = results['candidate_list_light'] diff --git a/candidate/models.py b/candidate/models.py index 8b1de8da8..5126adabc 100644 --- a/candidate/models.py +++ b/candidate/models.py @@ -198,7 +198,7 @@ def retrieve_all_candidates_for_office(self, office_id=0, office_we_vote_id='', candidate_query = candidate_query.filter(we_vote_id__in=candidate_we_vote_id_list) candidate_query = candidate_query.exclude(do_not_display_on_ballot=True) candidate_query = candidate_query.order_by('-twitter_followers_count') - candidate_list = candidate_query + candidate_list = list(candidate_query) if len(candidate_list): candidate_list_found = True @@ -214,8 +214,8 @@ def retrieve_all_candidates_for_office(self, office_id=0, office_we_vote_id='', status += 'FAILED retrieve_all_candidates_for_office ' + str(e) + ' ' success = False - for one_candidate in candidate_list: - candidate_we_vote_id_list.append(one_candidate.we_vote_id) + # for one_candidate in candidate_list: + # candidate_we_vote_id_list.append(one_candidate.we_vote_id) results = { 'success': success, @@ -233,7 +233,8 @@ def retrieve_all_candidates_for_upcoming_election( google_civic_election_id_list=[], state_code='', search_string=False, - return_list_of_objects=False): + return_list_of_objects=False, + read_only=False): candidate_list_objects = [] candidate_list_light = [] candidate_list_found = False @@ -256,7 +257,10 @@ def retrieve_all_candidates_for_upcoming_election( office_we_vote_id_list_by_candidate_we_vote_id = results['office_we_vote_id_list_by_candidate_we_vote_id'] try: - candidate_query = CandidateCampaign.objects.all() + if positive_value_exists(read_only): + candidate_query = CandidateCampaign.objects.using('readonly').all() + else: + candidate_query = CandidateCampaign.objects.all() if positive_value_exists(google_civic_election_id_list) and len(google_civic_election_id_list): candidate_query = candidate_query.filter(we_vote_id__in=candidate_we_vote_id_list) if positive_value_exists(state_code): @@ -341,13 +345,15 @@ def retrieve_all_candidates_for_one_year( candidate_year=0, limit_to_this_state_code='', search_string=False, - return_list_of_objects=False): + return_list_of_objects=False, + read_only=False): """ This might generate different results than retrieve_candidate_we_vote_id_list_from_year_list. :param candidate_year: :param limit_to_this_state_code: :param search_string: :param return_list_of_objects: + :param read_only: :return: """ candidate_list_objects = [] @@ -390,7 +396,10 @@ def retrieve_all_candidates_for_one_year( office_we_vote_id_list_by_candidate_we_vote_id = results['office_we_vote_id_list_by_candidate_we_vote_id'] try: - candidate_query = CandidateCampaign.objects.all() + if positive_value_exists(read_only): + candidate_query = CandidateCampaign.objects.using('readonly').all() + else: + candidate_query = CandidateCampaign.objects.all() if positive_value_exists(candidate_year_integer): candidate_query = candidate_query.filter(candidate_year=candidate_year_integer) if positive_value_exists(limit_to_this_state_code): @@ -656,7 +665,7 @@ def retrieve_candidate_count_for_office(self, office_id=0, office_we_vote_id='') candidate_list_manager = CandidateListManager() results = candidate_list_manager.retrieve_all_candidates_for_office( - office_id=office_id, office_we_vote_id=office_we_vote_id) + office_id=office_id, office_we_vote_id=office_we_vote_id, read_only=True) if not positive_value_exists(results['success']): status += 'RETRIEVE_CANDIDATE_COUNT_FOR_OFFICE_FAILED ' status += results['status'] @@ -789,7 +798,7 @@ def retrieve_candidates_from_all_elections_list(self): """ This is used by the admin tools to show CandidateCampaigns in a drop-down for example """ - candidates_list_temp = CandidateCampaign.objects.all() + candidates_list_temp = CandidateCampaign.objects.using('readonly').all() # Order by candidate_name. # To order by last name we will need to make some guesses in some case about what the last name is. candidates_list_temp = candidates_list_temp.order_by('candidate_name')[:300] @@ -814,7 +823,8 @@ def retrieve_possible_duplicate_candidates(self, candidate_name, google_civic_ca politician_we_vote_id, candidate_twitter_handle, ballotpedia_candidate_id, vote_smart_id, maplight_id, - we_vote_id_from_master=''): + we_vote_id_from_master='', + read_only=True): """ retrieve_possible_duplicate_candidates is used primarily to avoid duplicate candidate imports. :param candidate_name: @@ -829,6 +839,7 @@ def retrieve_possible_duplicate_candidates(self, candidate_name, google_civic_ca :param vote_smart_id: :param maplight_id: :param we_vote_id_from_master: + :param read_only: :return: """ candidate_list_objects = [] @@ -839,7 +850,10 @@ def retrieve_possible_duplicate_candidates(self, candidate_name, google_civic_ca office_manager = ContestOfficeManager() try: - candidate_query = CandidateCampaign.objects.all() + if positive_value_exists(read_only): + candidate_query = CandidateCampaign.objects.using('readonly').all() + else: + candidate_query = CandidateCampaign.objects.all() google_civic_election_id_list = [convert_to_int(google_civic_election_id)] results = self.retrieve_candidate_we_vote_id_list_from_election_list( @@ -1467,7 +1481,7 @@ def fetch_candidates_from_non_unique_identifiers_count( if keep_looking_for_duplicates and positive_value_exists(candidate_twitter_handle): try: - candidate_query = CandidateCampaign.objects.all() + candidate_query = CandidateCampaign.objects.using('readonly').all() candidate_query = candidate_query.filter(candidate_twitter_handle__iexact=candidate_twitter_handle) # Only look for matches in candidates in the specified elections, or in the year(s) the elections are in @@ -1492,7 +1506,7 @@ def fetch_candidates_from_non_unique_identifiers_count( if keep_looking_for_duplicates and positive_value_exists(candidate_name): # Search by Candidate name exact match try: - candidate_query = CandidateCampaign.objects.all() + candidate_query = CandidateCampaign.objects.using('readonly').all() candidate_query = candidate_query.filter(candidate_name__iexact=candidate_name) # Only look for matches in candidates in the specified elections, or in the year(s) the elections are in @@ -1516,7 +1530,7 @@ def fetch_candidates_from_non_unique_identifiers_count( if keep_looking_for_duplicates and positive_value_exists(candidate_name): # Search for Candidate(s) that contains the same first and last names try: - candidate_query = CandidateCampaign.objects.all() + candidate_query = CandidateCampaign.objects.using('readonly').all() # Only look for matches in candidates in the specified elections, or in the year(s) the elections are in candidate_query = candidate_query.filter( @@ -1627,7 +1641,8 @@ def retrieve_candidate_we_vote_id_list_from_election_list( if len(google_civic_election_id_list) > 0: results = self.retrieve_candidate_to_office_link_list( google_civic_election_id_list=google_civic_election_id_list, - state_code=limit_to_this_state_code) + state_code=limit_to_this_state_code, + read_only=True) if not positive_value_exists(results['success']): status += results['status'] success = False @@ -1689,13 +1704,14 @@ def retrieve_candidate_we_vote_id_list_from_year_list( success = True candidate_we_vote_id_list = [] - candidate_query = CandidateCampaign.objects.all() + candidate_query = CandidateCampaign.objects.using('readonly').all() candidate_query = candidate_query.filter(candidate_year__in=year_list) if positive_value_exists(limit_to_this_state_code): candidate_query = candidate_query.filter(state_code__iexact=limit_to_this_state_code) try: - candidate_we_vote_id_list = candidate_query.values_list('we_vote_id', flat=True).distinct() + candidate_query = candidate_query.values_list('we_vote_id', flat=True).distinct() + candidate_we_vote_id_list = list(candidate_query) except Exception as e: status += 'ERROR_WITH_DATABASE_QUERY: ' + str(e) + ' ' success = False @@ -1708,7 +1724,7 @@ def retrieve_candidate_we_vote_id_list_from_year_list( return results def fetch_candidate_we_vote_id_list_from_office_we_vote_id(self, office_we_vote_id): - results = self.retrieve_all_candidates_for_office(office_we_vote_id=office_we_vote_id) + results = self.retrieve_all_candidates_for_office(office_we_vote_id=office_we_vote_id, read_only=True) if not positive_value_exists(results['candidate_list_found']): return [] candidate_list = results['candidate_list'] @@ -1728,7 +1744,7 @@ def fetch_candidate_we_vote_id_list_from_election_list( return candidate_we_vote_id_list def fetch_office_we_vote_id_list_from_candidate_we_vote_id(self, candidate_we_vote_id): - results = self.retrieve_all_offices_for_candidate(candidate_we_vote_id=candidate_we_vote_id) + results = self.retrieve_all_offices_for_candidate(candidate_we_vote_id=candidate_we_vote_id, read_only=True) if not positive_value_exists(results['office_list_found']): return [] office_list = results['office_list'] @@ -1765,11 +1781,21 @@ def retrieve_google_civic_election_id_list_from_candidate_we_vote_id_list( } return results - def search_candidates_in_specific_elections(self, google_civic_election_id_list, search_string='', state_code='', - candidate_name='', candidate_twitter_handle='', - candidate_website='', candidate_email='', - candidate_facebook='', candidate_instagram='', twitter_handle_list='', - facebook_page_list='', exact_match=False): + def search_candidates_in_specific_elections( + self, + google_civic_election_id_list, + search_string='', + state_code='', + candidate_name='', + candidate_twitter_handle='', + candidate_website='', + candidate_email='', + candidate_facebook='', + candidate_instagram='', + twitter_handle_list='', + facebook_page_list='', + exact_match=False, + read_only=False): """ This function, search_candidates_in_specific_elections, is meant to cast a wider net for any possible candidates that might match. @@ -1814,7 +1840,10 @@ def search_candidates_in_specific_elections(self, google_civic_election_id_list, success = False candidate_we_vote_id_list = results['candidate_we_vote_id_list'] - candidate_query = CandidateCampaign.objects.all() + if positive_value_exists(read_only): + candidate_query = CandidateCampaign.objects.using('readonly').all() + else: + candidate_query = CandidateCampaign.objects.all() candidate_query = candidate_query.filter(we_vote_id__in=candidate_we_vote_id_list) if positive_value_exists(state_code): candidate_query = candidate_query.filter(state_code__iexact=state_code) @@ -1963,7 +1992,7 @@ def search_candidates_in_specific_elections(self, google_civic_election_id_list, } return results - def retrieve_candidates_with_misformatted_names(self, start=0, count=15): + def retrieve_candidates_with_misformatted_names(self, start=0, count=15, read_only=True): """ Get the first 15 records that have 3 capitalized letters in a row, as long as those letters are not 'III' i.e. King Henry III. Also exclude the names where the word "WITHDRAWN" has been appended when @@ -1974,9 +2003,13 @@ def retrieve_candidates_with_misformatted_names(self, start=0, count=15): :param start: :param count: + :param read_only: :return: """ - candidate_query = CandidateCampaign.objects.all() + if positive_value_exists(read_only): + candidate_query = CandidateCampaign.objects.using('readonly').all() + else: + candidate_query = CandidateCampaign.objects.all() # Get all candidates that have three capital letters in a row in their name, but exclude III (King Henry III) candidate_query = candidate_query.filter(candidate_name__regex=r'.*?[A-Z][A-Z][A-Z].*?(? -1. Start xcode (you can find it with Spotlight, or in the Application folder) +3. Start xcode (you can find it with Spotlight, or in the Application folder) -1. When prompted, download the "Additional Components" (the Command Line Tools). This takes many minutes to complete. -1. When you get to "Welcome to Xcode", quit out of the app. (For the WeVoteServer, we only need the command line tools that +4. July 2022, this step happens without a prompt: When prompted, download the "Additional Components" (the Command Line Tools). This takes many minutes to complete. + +5. When you get to "Welcome to Xcode", quit out of the app. (For the WeVoteServer, we only need the command line tools that come with Xcode.) -1. Navigate in Chrome to [GitHub](https://GitHub.com). Create a personal account if you don't already have one. +6. Navigate in Chrome to [GitHub](https://GitHub.com). Create a personal account if you don't already have one. -1. Within the GitHub site, navigate to [https://GitHub.com/wevote/WeVoteServer](https://GitHub.com/wevote/WeVoteServer). +7. Within the GitHub site, navigate to [https://GitHub.com/wevote/WeVoteServer](https://GitHub.com/wevote/WeVoteServer). Create a fork of wevote/WeVoteServer.git by selecting the "Fork" button (in the upper right of screen). -1. Download and install the Community version of PyCharm, it's free! + +8. Download and install the Community version of PyCharm, it's free! (If you are a student, you can get PyCharem Professional for free. Professional is nice, but not necessary.) [https://www.jetbrains.com/pycharm/download/#section=mac](https://www.jetbrains.com/pycharm/download/#section=mac) -1. StartPyCharm, and press the 'Get from VCS' button. +9. StartPyCharm, and press the 'Get from VCS' button. -1. Clone your fork of the git repository, by copying the URL to the repository into the URL filed, then press the Clone button. +10. Clone your fork of the git repository, by copying the URL to the repository into the URL filed, then press the Clone button. _What this means in english is that you have created a copy in GitHub of the WeVoteServer codebase, and cloning it downloads a copy of your copy to your Mac. At this instant, the 'develop' branch of wevote/WeVoteServer matches - your branch (in this example) SailingSteve/WeVoteServer and also matches the code on your Mac. + your branch (in this example) SailingSteve/WeVoteServer and also matches the code on your Mac. - + -1. The PyCharm IDE appears in 'Dracula' mode, with the repository loaded to your disk, and ready to edit. +11. The PyCharm IDE appears in 'Dracula' mode, with the repository loaded to your disk, and ready to edit. - + -1. If you like 'Dracula' mode, you can skip this step. Open PyCharm/Preferences and press the +12. If you like 'Dracula' mode, you can skip this step. Open PyCharm/Preferences and press the 'Sync with OS' button to match the display mode of your Mac. - - + + -1. In PyCharm/Preferences/Plugins enable the Markdown and IdeaVim tools (this takes a while). + +13. In PyCharm/Preferences/Plugins enable the IdeaVim tool (this takes a while). Feel free to add any other PyCharm tools that you would like! When done press 'Ok', and the IDE will reboot. - + + +14. If you are using one of the newer Macs with Apple Silicon processor, he installer offers the "Apple Silicon Version" which is better and more stable -- take it if it is offered! -1. If the Apple top menu, shows "Git" skip this step. If it says "VCS", the follow this step to configure Git +15. If the Apple top menu, shows "Git" skip this step. If it says "VCS", the follow this step to configure Git - + - Select 'Git' on the VCS meu, and press Ok. + Select 'Git' on the VCS meu, and press Ok. - + -1. In PyCharm set your git remotes. Navigate to the Git/'Manage Remotes...' dialog +16. In PyCharm set your git remotes. Navigate to the Git/'Manage Remotes...' dialog (July 2022: these two images might be reversed! (Verify!) But the results in the next step are corect.) - ![ScreenShot](images/RemotesUpstream.png) + ![ScreenShot](images/RemotesUpstream.png) - The WeVoteServer project defines upstream and origin differently than most projects. + The WeVoteServer project defines upstream and origin differently than most projects. - Click the edit (pencil) icon, and change the word origin to upstream. This is how it looks after the change. + Click the edit (pencil) icon, and change the word origin to upstream. This is how it looks after the change. - ![ScreenShot](images/RemotesOrigin.png) + ![ScreenShot](images/RemotesOrigin.png) -1. Then add a remote for your private branch by pressing the '+' button on the Git Remotes dialog. Add the url for your - fork of the WeVoteServer project origin (copy the url from the GitHub website). In this example, the developer - is "SailingSteve". +17. Then add a remote for your private branch by pressing the '+' button on the Git Remotes dialog. Add the url for your + fork of the WeVoteServer project origin (copy the url from the GitHub website). In this example, the developer + is "SailingSteve". - ![ScreenShot](images/AddUpstream2021.png) - -1. When the cloning is complete, it will look something like this. + ![ScreenShot](images/AddUpstream2021.png) +18. When the cloning is complete, it will look something like this. - ![ScreenShot](images/CorrectOrigin2021.png) + ![ScreenShot](images/CorrectOrigin2021.png) - Press Ok to close the dialog + Press Ok to close the dialog -1. In PyCharm copy `environment_variables-template.json` to `environment_variables.json` +19. In PyCharm copy `environment_variables-template.json` to `environment_variables.json` - ![ScreenShot](images/PyCharmTemplateCopy2021.png) + ![ScreenShot](images/PyCharmTemplateCopy2021.png) - Right click on `environment_variables-template.json` and select 'Copy', then right click paste on the `config` - directory and select 'Paste' in the pop-up, and then in the copy dialog that open up, and change the "new name:" to - `environment_variables.json` + Right click on `environment_variables-template.json` and select 'Copy', then right click paste on the `config` + directory and select 'Paste' in the pop-up, and then in the copy dialog that open up, and change the "new name:" to + `environment_variables.json` - If you skip this step, in a much later step, when you run "makemigrations", it will fail with an - 'Unable to set the **** variable from "os.environ" or JSON file' error. + If you skip this step, in a much later step, when you run "makemigrations", it will fail with an + 'Unable to set the **** variable from "os.environ" or JSON file' error. - **There are a number of secret values in `environment_variables.json` that are not in source control, - you will need to check in with Dale, as you find that you need them.** + **There are a number of secret values in `environment_variables.json` that are not in source control, + you will need to check in with Dale, as you find that you need them.** -1. In PyCharm, open the Terminal window and accept use of the z shell (if you want to use some other shell, feel free to skip this step). +20. In PyCharm, open the Terminal window and accept use of the z shell (if you want to use some other shell, feel free to skip this step). - ![ScreenShot](images/AcceptZShell.png) + ![ScreenShot](images/AcceptZShell.png) - The terminal opens up with the project root directory set as the pwd (which is handy). + The terminal opens up with the project root directory set as the pwd (which is handy). -1. In the PyCharm terminal window download [Homebrew]( https://brew.sh/) ("the missing package manager for macOS") by entering +21. In the PyCharm terminal window download [Homebrew]( https://brew.sh/) ("the missing package manager for macOS") by entering the following command: - ``` - $ /bin/bash -c "$(curl -fsSL https://raw.GitHubusercontent.com/Homebrew/install/master/install.sh)" - ``` - - This loads and runs a Ruby script (Ruby comes pre-installed in macOS), and Ruby uses curl (also pre-loaded) to pull the file - into the bash (terminal) command shell for execution. This Ruby script also internally uses 'sudo' which temporarily gives - the script root privileges to install software, so you will need to know an admin password for your Mac. - - This script can take a few minutes to complete. - -2. Install the latest version of Python - - ``` - $ brew install python - ``` - If an older version of Python has been installed, and the installation fails, you will see the following error: - ``` - Error: python@3.9 3.9.1_1 is already installed - To upgrade to 3.9.5, run: - brew upgrade python@3.9 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % - ``` - In which case you run the suggested upgrade command, in this example it would be `brew upgrade python@3.9`, then finally export the path as shown below. - ``` - $ export PATH="/usr/local/opt/python/libexec/bin:$PATH" - ``` -3. Test that the newly installed Python is in the path. macOS comes with Python 2 preinstalled, so + ``` + $ /bin/bash -c "$(curl -fsSL https://raw.GitHubusercontent.com/Homebrew/install/master/install.sh)" + ``` + + This loads and runs a Ruby script (Ruby comes pre-installed in macOS), and Ruby uses curl (also pre-loaded) to pull the file + into the bash (terminal) command shell for execution. This Ruby script also internally uses 'sudo' which temporarily gives + the script root privileges to install software, so you will need to know an admin password for your Mac. + + This script can take a few minutes to complete. + +22. Install the latest version of Python + + ``` + $ brew install python + ``` + If an older version of Python has been installed, and the installation fails, you will see the following error: + ``` + Error: python@3.9 3.9.1_1 is already installed + To upgrade to 3.9.5, run: + brew upgrade python@3.9 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % + ``` + In which case you run the suggested upgrade command, in this example it would be `brew upgrade python@3.9`, then finally export the path as shown below. + ``` + $ export PATH="/usr/local/opt/python/libexec/bin:$PATH" + ``` +23. Test that the newly installed Python is in the path. macOS comes with Python 2 preinstalled, so if the reported version is 2, then add the newly loaded python to the path with the export command. Then confirm that the default python is now version 3.9 or later. (Version 3.6 has problems with macOS Big Sur or later) + ``` + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version + Python 2.7.16 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % export PATH="/usr/local/opt/python/libexec/bin:$PATH" + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version + Python 3.9.5 + Steve@Vickies-MacBook-Pro-2037 WeVoteServer % + ``` + 2021: For an 'Apple M1 Max' ARM-64 Processor... + ``` + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version + Python 2.7.18 + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % export PATH="/opt/homebrew/opt/python@3.9/libexec/bin:$PATH" + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version + Python 3.9.9 + stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % + ``` + Note July 2022: n the homebrew directory (not in the venv), had to make a symlink between python3 and python so that psycopg2-binary could find python. (To be verified) + +24. If python --version fails, + try ``` - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version - Python 2.7.16 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % export PATH="/usr/local/opt/python/libexec/bin:$PATH" - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % python --version - Python 3.9.5 - Steve@Vickies-MacBook-Pro-2037 WeVoteServer % - ``` - 2021: For an 'Apple M1 Max' ARM-64 Processor... - ``` - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version - Python 2.7.18 - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % export PATH="/opt/homebrew/opt/python@3.9/libexec/bin:$PATH" - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % python --version - Python 3.9.9 - stevepodell@Steves-MBP-M1-Dec2021 WeVoteServer % + ln -s /opt/homebrew/bin/python3 /opt/homebrew/bin/python ``` +needed to install postgres before the requirements because psyco3-3 binary requires pg_config which is not installed yet. -5. Set up a Virtual Environment with the new Python Interpreter. +25. Set up a Virtual Environment with the new Python Interpreter. Navigate to: PyCharm/Preferences/Project: WeVoteServer/Python Interpreter. - + -6. Click the Gear icon, then select "Add". PyCharm will detect the latest interpreter from the PATH environment variable, - and pre-populate the dialog. Check the two checkboxes `Inherit global site-packages` and `make available to all projects`. +26. Click the Gear icon, then select "Add". PyCharm will detect the latest interpreter from the PATH environment variable, + and pre-populate the dialog. Check the two checkboxes `Inherit global site-packages` and `make available to all projects`. - + - Confirm that the 'Base interpreter' field shows us using the Python version that you just downloaded, and it knows the location for pip, setuptools, and wheel (3 python utilities). - Then press Ok. + Confirm that the 'Base interpreter' field shows us using the Python version that you just downloaded, and it knows the location for pip, setuptools, and wheel (3 python utilities). + Then press Ok. - ![ScreenShot](images/VenvCompleted.png) + ![ScreenShot](images/VenvCompleted.png) -7. Confirm that the new virtual environment is in effect, by closing all open Terminal windows within +27. Confirm that the new virtual environment is in effect, by closing all open Terminal windows within PyCharm and opening a new one. - + - If you see '(venv)' at the beginning of the command line, all is well. + If you see '(venv)' at the beginning of the command line, all is well. -8. Install OpenSSL, the pyopenssl and https clients: +28. Install OpenSSL, the pyopenssl and https clients: - `(WeVoteServerPy3.7) $ brew install openssl` - If it is already installed, no worries! + `(WeVoteServerPy3.7) $ brew install openssl` + If it is already installed, no worries! -9. Link libssl and libcrypto so that pip can find them: - ``` - $ ln -s /usr/local/opt/openssl/lib/libcrypto.dylib /usr/local/lib/libcrypto.dylib - $ ln -s /usr/local/opt/openssl/lib/libssl.dylib /usr/local/lib/libssl.dylib - ``` -10. Install libmagic +29. Link libssl and libcrypto so that pip can find them: + ``` + $ ln -s /usr/local/opt/openssl/lib/libcrypto.dylib /usr/local/lib/libcrypto.dylib + $ ln -s /usr/local/opt/openssl/lib/libssl.dylib /usr/local/lib/libssl.dylib + ``` +30. Install libmagic `(WeVoteServerPy3.7) $ brew install libmagic` -11. Install all the other Python packages required by the WeVoteServer project (there are a lot of them!) +31. Install all the other Python packages required by the WeVoteServer project (there are a lot of them!) `(WeVoteServer3.7) $ pip3 install -r requirements.txt` @@ -220,6 +232,8 @@ PyCharm and opening a new one. linux/macOS binary libraries based on c language packages and compiled with gcc. Wheels allow python library developers to speed up execution by coding critical or complex sections the c language. Interpreted Python code runs slower than compiled c. + + **Note July 2022 if this fails due to `psycopg2-binary` requiring `pg_config` (which is installed with postgres), install Postgres first then come back and do the pip3 install -r requirements.txt` command.** If this installation succeeds with no missing libraries, or other compiler errors, we are almost done. If this installation fails, please ask for help. @@ -264,7 +278,7 @@ this step. To see if postgres is already running, check with lsof in a terminal `(venv) $ brew services start postgresql` -5. Create a default database, and a default user, and then log into the 'psql postgres' PostgreSQL command interpreter: +5. Create a default database, and a default user, and then log into the 'psql postgres' PostgreSQL command interpreter ("postgres=#" is the command prompt, you should not have to type this in): _New way: November 2021, using Postgres 14.0_ ``` @@ -273,24 +287,22 @@ this step. To see if postgres is already running, check with lsof in a terminal Type "help" for help. postgres=# createdb - postgres-# createuser -s postgres - postgres-# \du + postgres=# createuser -s postgres (TODO 7/8: CREATE ROLE postgres WITH SUPERUSER CREATEDB CREATEROLE LOGIN ENCRYPTED PASSWORD ‘stevePG’; + postgres=# \du List of roles Role name | Attributes | Member of -------------+------------------------------------------------------------+----------- stevepodell | Superuser, Create role, Create DB, Replication, Bypass RLS | {} - postgres-# postgres=# create database WeVoteServerDB; CREATE DATABASE postgres=# grant all privileges on database WeVoteServerDB to postgres; GRANT - postgres=# postgres=# \l List of databases Name | Owner | Encoding | Collate | Ctype | Access privileges ----------------+-------------+----------+---------+-------+----------------------------- - WeVoteServerDB | stevepodell | UTF8 | C | C | + WeVoteServerDB | stevepodell | UTF8 | C | C | (TODO 7/8/22 these don't match the latest setup) postgres | stevepodell | UTF8 | C | C | template0 | stevepodell | UTF8 | C | C | =c/stevepodell + | | | | | stevepodell=CTc/stevepodell @@ -344,10 +356,10 @@ this step. To see if postgres is already running, check with lsof in a terminal This can take a few minutes to complete. When `brew install --cask pgadmin4` finishes, it prints out `Moving App 'pgAdmin 4.app' to '/Applications/pgAdmin 4.app'.` The latest pgAdmin4 has a webapp architecture (it is not a compiled program). The app you start from the Application folder is actually a - single purpose web server, and the UI for the app appears in Chrome as a local website. + single purpose web server, and the UI for the app appears. (7/8/22 needs a new ) 7. Use Spotlight to find and launch the pgAdmin4 app. Once launched, the pgAdmin4 webapp will display in a new tab within Chrome. - On that new tab, Right-click on "Servers" and choose "Create > Server" + On that new tab, click on the "Add new Servers" button and choose "Create > Server" @@ -360,7 +372,7 @@ this step. To see if postgres is already running, check with lsof in a terminal * Port: 5432 * Maintenance database: postgres * User name: postgres - * Password: + * Password: (mine is stevePG) * Save password: checked ![ScreenShot](images/CreateServerConnection2.png) @@ -462,7 +474,7 @@ this step. To see if postgres is already running, check with lsof in a terminal (WeVoteServer3.7) admin$ ``` -1. Navigate to [http://localhost:8000/admin/](http://localhost:8000/admin/) and sign in with your new username/password. +1. Navigate to [http://localhost:8000/admin/](http://localhost:8000/admin/) and sign in with your new username/password (for example mine is stevepodell/stevePG.). 1. Your local instance of the WeVoteServer is now setup and running (although there is no election data stored in your Postgres instance, for it to serve to clients at this point). diff --git a/docs/ServerOAuth.md b/docs/ServerOAuth.md index 27ed24967..87936d819 100644 --- a/docs/ServerOAuth.md +++ b/docs/ServerOAuth.md @@ -76,6 +76,8 @@ you need to add that domain to your 127.0.0.1 line in /etc/hosts. After the cha You will need to elevate your privileges with sudo to make this edit to this linux system file ... ` % sudo vi /etc/hosts` or with some other editor. +Note July 2022: The auto generated certificate that is made by runsslserver generates warnings in browsers (not really a problem), +but may stop the JavaScript builtin fetch() function from completing. The browser extension has to use fetch. ## Facebook diff --git a/elected_official/models.py b/elected_official/models.py index 06943a519..1b28d66f3 100644 --- a/elected_official/models.py +++ b/elected_official/models.py @@ -1989,7 +1989,7 @@ def count_elected_officials_for_election(self, google_civic_election_id): success = False if positive_value_exists(google_civic_election_id): try: - elected_official_item_queryset = ElectedOfficial.objects.all() + elected_official_item_queryset = ElectedOfficial.objects.using('readonly').all() elected_official_item_queryset = elected_official_item_queryset.filter( google_civic_election_id=google_civic_election_id) elected_officials_count = elected_official_item_queryset.count() diff --git a/election/models.py b/election/models.py index 255a4a718..9ea42ea64 100644 --- a/election/models.py +++ b/election/models.py @@ -80,7 +80,7 @@ def get_election_state(self): class Election(models.Model): # The unique ID of this election. (Provided by Google Civic) google_civic_election_id = models.CharField(verbose_name="google civic election id", - max_length=20, null=True, unique=True) + max_length=20, null=True, unique=True, db_index=True) google_civic_election_id_new = models.PositiveIntegerField( verbose_name="google civic election id", null=True, unique=False) # Make unique=True after data is migrated ballotpedia_election_id = models.PositiveIntegerField( @@ -101,16 +101,16 @@ class Election(models.Model): max_length=2, null=True, blank=True, db_index=True) # We generate a string with all the state codes. Ex/ CA,CO,UT state_code_list_raw = models.CharField(max_length=255, null=True, blank=True) - include_in_list_for_voters = models.BooleanField(default=False) + include_in_list_for_voters = models.BooleanField(default=False, db_index=True) # For internal notes regarding gathering data for this election internal_notes = models.TextField(null=True, blank=True, default=None) # Not an election we will be supporting - ignore_this_election = models.BooleanField(default=False) + ignore_this_election = models.BooleanField(default=False, db_index=True) # Have we finished all the election preparation related to Offices, Candidates, Measures and Ballot Locations? election_preparation_finished = models.BooleanField(default=False) - # Have we finished all of the election preparation related to Candidate photos? + # Have we finished all the election preparation related to Candidate photos? candidate_photos_finished = models.BooleanField(default=False) # This is a multi-state election @@ -433,7 +433,7 @@ def retrieve_elections_by_date(self, newest_to_oldest=True, include_test_electio election_list_query = Election.objects.using('readonly').all() if not positive_value_exists(include_test_election): election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') if newest_to_oldest: # Typically we want the newest election displayed first, with the older elections later in the list election_list_query = election_list_query.reverse() @@ -528,7 +528,7 @@ def retrieve_election( def retrieve_listed_elections(self): """ - These are all of the elections marked as "listed" with "include_in_list_for_voters" + These are all the elections marked as "listed" with "include_in_list_for_voters" :return: """ election_list = [] @@ -557,14 +557,18 @@ def retrieve_upcoming_elections( state_code="", without_state_code=False, require_include_in_list_for_voters=False, - include_test_election=False): + include_test_election=False, + read_only=True): status = '' election_list_found = False upcoming_election_list = [] today = datetime.now().date() we_vote_date_string = convert_date_to_we_vote_date_string(today) try: - election_list_query = Election.objects.using('readonly').all() + if positive_value_exists(read_only): + election_list_query = Election.objects.using('readonly').all() + else: + election_list_query = Election.objects.all() election_list_query = election_list_query.filter(election_day_text__gte=we_vote_date_string) election_list_query = election_list_query.exclude(ignore_this_election=True) if positive_value_exists(require_include_in_list_for_voters): @@ -575,7 +579,7 @@ def retrieve_upcoming_elections( election_list_query = election_list_query.filter(state_code__iexact=state_code) if not positive_value_exists(include_test_election): election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') upcoming_election_list = list(election_list_query) @@ -604,7 +608,8 @@ def retrieve_upcoming_google_civic_election_id_list( upcoming_google_civic_election_id_list = [] results = self.retrieve_upcoming_elections( state_code=limit_to_this_state_code, - require_include_in_list_for_voters=require_include_in_list_for_voters) + require_include_in_list_for_voters=require_include_in_list_for_voters, + read_only=True) if results['election_list_found']: election_list = results['election_list'] for one_election in election_list: @@ -655,7 +660,7 @@ def retrieve_prior_elections_this_year(self, state_code="", without_state_code=F elif positive_value_exists(state_code): election_list_query = election_list_query.filter(state_code__iexact=state_code) election_list_query = election_list_query.exclude(google_civic_election_id=2000) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') prior_election_list = list(election_list_query) @@ -1105,7 +1110,7 @@ def retrieve_elections_by_election_date(self, election_day_text='', include_test election_list_query = election_list_query.exclude(google_civic_election_id=2000) if election_day_text: election_list_query = election_list_query.filter(election_day_text=election_day_text) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') election_list = election_list_query status = 'ELECTIONS_FOUND' success = True @@ -1185,7 +1190,7 @@ def retrieve_elections_by_state_and_election_date(self, state_code='', election_ if state_code and election_day_text: election_list_query = election_list_query.filter(state_code__iexact=state_code, election_day_text=election_day_text) - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') election_list = election_list_query status = 'ELECTIONS_FOUND' success = True diff --git a/election/views_admin.py b/election/views_admin.py index 6e17eafc8..502550e1b 100644 --- a/election/views_admin.py +++ b/election/views_admin.py @@ -43,7 +43,7 @@ from measure.models import ContestMeasure, ContestMeasureListManager from office.models import ContestOffice, ContestOfficeListManager, ContestOfficeManager from pledge_to_vote.models import PledgeToVoteManager -from polling_location.models import PollingLocation +from polling_location.models import PollingLocation, PollingLocationManager from position.models import ANY_STANCE, PositionEntered, PositionForFriends, PositionListManager import pytz from quick_info.models import QuickInfoManager @@ -864,7 +864,7 @@ def election_list_view(request): office_manager = ContestOfficeManager() election_list_query = Election.objects.all() - election_list_query = election_list_query.order_by('election_day_text') + election_list_query = election_list_query.order_by('election_day_text', 'election_name') election_list_query = election_list_query.exclude(google_civic_election_id=2000) if positive_value_exists(show_ignored_elections): # Do not filter out ignored elections @@ -1048,6 +1048,7 @@ def election_list_view(request): # Upcoming refresh date scheduled? refresh_date_added_to_queue = None + retrieve_date_added_to_queue = None try: batch_process_queryset = BatchProcess.objects.all() batch_process_queryset = \ @@ -1055,7 +1056,8 @@ def election_list_view(request): batch_process_queryset = batch_process_queryset.filter(date_completed__isnull=True) batch_process_queryset = batch_process_queryset.exclude(batch_process_paused=True) ballot_item_processes = [ - 'REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS'] + 'REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS', + 'RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS'] batch_process_queryset = batch_process_queryset.filter(kind_of_process__in=ballot_item_processes) batch_process_queryset = batch_process_queryset.order_by("-id") @@ -1067,6 +1069,9 @@ def election_list_view(request): if one_batch_process.kind_of_process == 'REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS': if not refresh_date_added_to_queue and one_batch_process.date_added_to_queue: refresh_date_added_to_queue = one_batch_process.date_added_to_queue + elif one_batch_process.kind_of_process == 'RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS': + if not retrieve_date_added_to_queue and one_batch_process.date_added_to_queue: + retrieve_date_added_to_queue = one_batch_process.date_added_to_queue except BatchProcess.DoesNotExist: # No offices found. Not a problem. batch_process_list = [] @@ -1079,6 +1084,7 @@ def election_list_view(request): election.refresh_date_added_to_queue = refresh_date_added_to_queue election.retrieve_date_completed = retrieve_date_completed election.retrieve_date_started = retrieve_date_started + election.retrieve_date_added_to_queue = retrieve_date_added_to_queue if refresh_date_completed: most_recent_time = refresh_date_completed @@ -1594,6 +1600,10 @@ def election_summary_view(request, election_local_id=0, google_civic_election_id ballot_returned_list_manager = BallotReturnedListManager() candidate_list_manager = CandidateListManager() office_manager = ContestOfficeManager() + # See if the number of map points for this state exceed the "large" threshold + polling_location_manager = PollingLocationManager() + map_points_retrieved_each_batch_chunk = \ + polling_location_manager.calculate_number_of_map_points_to_retrieve_with_each_batch_chunk(state_code) if election_found: batch_manager = BatchManager() @@ -1825,6 +1835,7 @@ def election_summary_view(request, election_local_id=0, google_civic_election_id 'election': election, 'google_civic_election_id': google_civic_election_id, 'is_national_election': election.is_national_election, + 'map_points_retrieved_each_batch_chunk': map_points_retrieved_each_batch_chunk, 'messages_on_stage': messages_on_stage, 'more_than_three_ballotpedia_elections': more_than_three_ballotpedia_elections, 'all_ballotpedia_elections_shown': all_ballotpedia_elections_shown, @@ -1845,6 +1856,7 @@ def election_summary_view(request, election_local_id=0, google_civic_election_id 'entries_missing_latitude_longitude': entries_missing_latitude_longitude, 'google_civic_election_id': google_civic_election_id, 'is_national_election': is_national_election, + 'map_points_retrieved_each_batch_chunk': map_points_retrieved_each_batch_chunk, 'messages_on_stage': messages_on_stage, 'state_code': state_code, 'state_list': sorted_state_list, @@ -2100,11 +2112,12 @@ def election_migration_view(request): from_election_ballot_item_count = 0 try: if positive_value_exists(from_state_code): - from_election_ballot_item_count = BallotItem.objects.filter(google_civic_election_id=from_election_id)\ + from_election_ballot_item_count = BallotItem.objects.using('readonly')\ + .filter(google_civic_election_id=from_election_id)\ .filter(state_code__iexact=from_state_code).count() else: from_election_ballot_item_count = \ - BallotItem.objects.filter(google_civic_election_id=from_election_id).count() + BallotItem.objects.using('readonly').filter(google_civic_election_id=from_election_id).count() if positive_value_exists(change_now): if positive_value_exists(from_state_code): BallotItem.objects.filter(google_civic_election_id=from_election_id)\ @@ -2253,11 +2266,13 @@ def election_migration_view(request): contest_measure_query = ContestMeasure.objects.filter(google_civic_election_id=from_election_id)\ .filter(state_code__iexact=from_state_code) from_election_measure_count = contest_measure_query.count() - contest_measure_we_vote_ids_migrated = contest_measure_query.values_list('we_vote_id', flat=True).distinct() + contest_measure_query = contest_measure_query.values_list('we_vote_id', flat=True).distinct() + contest_measure_we_vote_ids_migrated = list(contest_measure_query) else: contest_measure_query = ContestMeasure.objects.filter(google_civic_election_id=from_election_id) from_election_measure_count = contest_measure_query.count() - contest_measure_we_vote_ids_migrated = contest_measure_query.values_list('we_vote_id', flat=True).distinct() + contest_measure_query = contest_measure_query.values_list('we_vote_id', flat=True).distinct() + contest_measure_we_vote_ids_migrated = list(contest_measure_query) if positive_value_exists(change_now): if positive_value_exists(from_state_code): ContestMeasure.objects.filter(google_civic_election_id=from_election_id)\ @@ -2280,12 +2295,14 @@ def election_migration_view(request): contest_office_query = ContestOffice.objects.filter(google_civic_election_id=from_election_id)\ .filter(state_code__iexact=from_state_code) from_election_office_count = contest_office_query.count() - contest_office_we_vote_ids_migrated = contest_office_query.values_list('we_vote_id', flat=True).distinct() + contest_office_query = contest_office_query.values_list('we_vote_id', flat=True).distinct() + contest_office_we_vote_ids_migrated = list(contest_office_query) else: contest_office_query = \ ContestOffice.objects.filter(google_civic_election_id=from_election_id) from_election_office_count = contest_office_query.count() - contest_office_we_vote_ids_migrated = contest_office_query.values_list('we_vote_id', flat=True).distinct() + contest_office_query = contest_office_query.values_list('we_vote_id', flat=True).distinct() + contest_office_we_vote_ids_migrated = list(contest_office_query) if positive_value_exists(change_now): if positive_value_exists(from_state_code): ContestOffice.objects.filter(google_civic_election_id=from_election_id)\ @@ -2330,12 +2347,14 @@ def election_migration_view(request): elected_office_query = ElectedOffice.objects.filter(google_civic_election_id=from_election_id)\ .filter(state_code__iexact=from_state_code) from_elected_office_count = elected_office_query.count() - elected_office_we_vote_ids_migrated = elected_office_query.values_list('we_vote_id', flat=True).distinct() + elected_office_query = elected_office_query.values_list('we_vote_id', flat=True).distinct() + elected_office_we_vote_ids_migrated = list(elected_office_query) else: elected_office_query = \ ElectedOffice.objects.filter(google_civic_election_id=from_election_id) from_elected_office_count = elected_office_query.count() - elected_office_we_vote_ids_migrated = elected_office_query.values_list('we_vote_id', flat=True).distinct() + elected_office_query = elected_office_query.values_list('we_vote_id', flat=True).distinct() + elected_office_we_vote_ids_migrated = list(elected_office_query) if positive_value_exists(change_now): if positive_value_exists(from_state_code): ElectedOffice.objects.filter(google_civic_election_id=from_election_id)\ @@ -2673,7 +2692,8 @@ def election_migration_view(request): # ######################################## # BatchRowActionBallotItem if not positive_value_exists(from_state_code): # Only move if we are NOT moving just one state - batch_query = BatchRowActionBallotItem.objects.filter(google_civic_election_id=from_election_id) + batch_query = BatchRowActionBallotItem.objects.using('readonly')\ + .filter(google_civic_election_id=from_election_id) we_vote_batch_count = batch_query.count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): try: @@ -2686,7 +2706,8 @@ def election_migration_view(request): # ######################################## # BatchRowActionCandidate if not positive_value_exists(from_state_code): # Only move if we are NOT moving just one state - batch_query = BatchRowActionCandidate.objects.filter(google_civic_election_id=from_election_id) + batch_query = BatchRowActionCandidate.objects.using('readonly')\ + .filter(google_civic_election_id=from_election_id) we_vote_batch_count = batch_query.count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): try: @@ -2699,7 +2720,8 @@ def election_migration_view(request): # ######################################## # BatchRowActionContestOffice if not positive_value_exists(from_state_code): # Only move if we are NOT moving just one state - batch_query = BatchRowActionContestOffice.objects.filter(google_civic_election_id=from_election_id) + batch_query = BatchRowActionContestOffice.objects.using('readonly')\ + .filter(google_civic_election_id=from_election_id) we_vote_batch_count = batch_query.count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): try: @@ -2712,7 +2734,7 @@ def election_migration_view(request): # ######################################## # BatchRowActionMeasure if not positive_value_exists(from_state_code): # Only move if we are NOT moving just one state - batch_query = BatchRowActionMeasure.objects.filter(google_civic_election_id=from_election_id) + batch_query = BatchRowActionMeasure.objects.using('readonly').filter(google_civic_election_id=from_election_id) we_vote_batch_count = batch_query.count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): try: @@ -2725,7 +2747,7 @@ def election_migration_view(request): # ######################################## # BatchRowActionPosition if not positive_value_exists(from_state_code): # Only move if we are NOT moving just one state - batch_query = BatchRowActionPosition.objects.filter(google_civic_election_id=from_election_id) + batch_query = BatchRowActionPosition.objects.using('readonly').filter(google_civic_election_id=from_election_id) we_vote_batch_count = batch_query.count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): try: @@ -2738,7 +2760,7 @@ def election_migration_view(request): # ######################################## # BatchRowTranslationMap if not positive_value_exists(from_state_code): # Only move if we are NOT moving just one state - batch_query = BatchRowTranslationMap.objects.filter(google_civic_election_id=from_election_id) + batch_query = BatchRowTranslationMap.objects.using('readonly').filter(google_civic_election_id=from_election_id) we_vote_batch_count = batch_query.count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): try: @@ -2752,10 +2774,12 @@ def election_migration_view(request): # BatchSet try: if positive_value_exists(from_state_code): - we_vote_batch_count = BatchSet.objects.filter(google_civic_election_id=from_election_id)\ + we_vote_batch_count = BatchSet.objects.using('readonly')\ + .filter(google_civic_election_id=from_election_id)\ .filter(state_code__iexact=from_state_code).count() else: - we_vote_batch_count = BatchSet.objects.filter(google_civic_election_id=from_election_id).count() + we_vote_batch_count = BatchSet.objects.using('readonly')\ + .filter(google_civic_election_id=from_election_id).count() if positive_value_exists(change_now) and positive_value_exists(we_vote_batch_count): if positive_value_exists(from_state_code): BatchSet.objects.filter(google_civic_election_id=from_election_id)\ diff --git a/email_outbound/controllers.py b/email_outbound/controllers.py index 65c4e08d9..f59773b28 100644 --- a/email_outbound/controllers.py +++ b/email_outbound/controllers.py @@ -216,6 +216,8 @@ def augment_emails_for_voter_with_we_vote_data(voter_we_vote_id=''): status = '' success = True + from friend.models import FriendManager + friend_manager = FriendManager() from voter.models import VoterManager voter_manager = VoterManager() # Augment all voter contacts with updated data from We Vote @@ -282,6 +284,36 @@ def augment_emails_for_voter_with_we_vote_data(voter_we_vote_id=''): except Exception as e: status += "FAILED_TO_UPDATE_VOTER_CONTACT_EMAIL: " + str(e) + ' ' + # Retrieve again now that voter_we_vote_id has been updated, so we can see if they are a friend + voter_contact_results = voter_manager.retrieve_voter_contact_email_list( + imported_by_voter_we_vote_id=voter_we_vote_id, + read_only=False) + if voter_contact_results['voter_contact_email_list_found']: + voter_contact_email_list = voter_contact_results['voter_contact_email_list'] + # Retrieve main voter's friends, and then update voter contacts with is_friend + friend_results = friend_manager.retrieve_friends_we_vote_id_list(voter_we_vote_id) + friends_we_vote_id_list = [] + if friend_results['friends_we_vote_id_list_found']: + friends_we_vote_id_list = friend_results['friends_we_vote_id_list'] + for voter_contact in voter_contact_email_list: + if positive_value_exists(voter_contact.voter_we_vote_id): + should_save_voter_contact = False + voter_contact_should_be_friend = voter_contact.voter_we_vote_id in friends_we_vote_id_list + if positive_value_exists(voter_contact.is_friend): + if voter_contact_should_be_friend: + pass # all is well! + else: + voter_contact.is_friend = False + should_save_voter_contact = True + elif voter_contact_should_be_friend: + voter_contact.is_friend = True + should_save_voter_contact = True + if should_save_voter_contact: + try: + voter_contact.save() + except Exception as e: + status += "COULD_NOT_SAVE_VOTER_CONTACT: " + str(e) + " " + results = { 'success': success, 'status': status, @@ -683,8 +715,12 @@ def schedule_email_with_email_outbound_description(email_outbound_description, s subject = email_template_results['subject'] message_text = email_template_results['message_text'] message_html = email_template_results['message_html'] - schedule_email_results = email_manager.schedule_email(email_outbound_description, subject, - message_text, message_html, send_status) + schedule_email_results = email_manager.schedule_email( + email_outbound_description=email_outbound_description, + subject=subject, + message_text=message_text, + message_html=message_html, + send_status=send_status) success = schedule_email_results['success'] status += schedule_email_results['status'] email_scheduled_saved = schedule_email_results['email_scheduled_saved'] @@ -754,15 +790,34 @@ def schedule_verification_email( subject = "Please verify your email" + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/login" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/login/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_LOGIN_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='login') + template_variables_for_json = { - "subject": subject, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_voter_email, - "we_vote_url": web_app_root_url_verified, + "subject": subject, "verify_email_link": web_app_root_url_verified + "/verify_email/" + recipient_email_address_secret_key, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", + "we_vote_url": web_app_root_url_verified, } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) verification_from_email = "We Vote " # TODO DALE Make system variable @@ -775,7 +830,10 @@ def schedule_verification_email( recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_voter_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " if outbound_results['email_outbound_description_saved']: email_outbound_description = outbound_results['email_outbound_description'] @@ -851,14 +909,33 @@ def schedule_link_to_sign_in_email( if is_cordova: link_to_sign_in = "wevotetwitterscheme://sign_in_email/" + recipient_email_address_secret_key + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/login" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/login/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_LOGIN_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='login') + template_variables_for_json = { - "subject": subject, + "link_to_sign_in": link_to_sign_in, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_voter_email, + "subject": subject, "we_vote_url": web_app_root_url_verified, - "link_to_sign_in": link_to_sign_in, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) verification_from_email = "We Vote " # TODO DALE Make system variable @@ -870,7 +947,10 @@ def schedule_link_to_sign_in_email( recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_voter_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " if outbound_results['email_outbound_description_saved']: email_outbound_description = outbound_results['email_outbound_description'] @@ -936,14 +1016,33 @@ def schedule_sign_in_code_email( subject = "Your Sign in Code" + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/login" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/login/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_LOGIN_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='login') + template_variables_for_json = { - "subject": subject, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_voter_email, - "we_vote_url": web_app_root_url_verified, "secret_numerical_code": secret_numerical_code, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", + "subject": subject, + "we_vote_url": web_app_root_url_verified, } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) verification_from_email = "We Vote " # TODO DALE Make system variable @@ -955,7 +1054,10 @@ def schedule_sign_in_code_email( recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_voter_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] if outbound_results['email_outbound_description_saved']: email_outbound_description = outbound_results['email_outbound_description'] @@ -1009,7 +1111,7 @@ def voter_email_address_retrieve_for_api(voter_device_id): # voterEmailAddressR return json_data voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=False) voter_id = voter_results['voter_id'] if not positive_value_exists(voter_id): error_results = { @@ -1108,7 +1210,7 @@ def voter_email_address_sign_in_for_api(voter_device_id, email_secret_key): # v email_manager = EmailManager() # Look to see if there is an EmailAddress entry for the incoming text_for_email_address or email_we_vote_id - email_results = email_manager.retrieve_email_address_object_from_secret_key(email_secret_key) + email_results = email_manager.retrieve_email_address_object_from_secret_key(email_secret_key=email_secret_key) if not email_results['email_address_object_found']: status += email_results['status'] error_results = { @@ -1200,7 +1302,7 @@ def voter_email_address_verify_for_api(voter_device_id, email_secret_key): # vo email_manager = EmailManager() # Look to see if there is an EmailAddress entry for the incoming text_for_email_address or email_we_vote_id - email_results = email_manager.verify_email_address_object_from_secret_key(email_secret_key) + email_results = email_manager.verify_email_address_object_from_secret_key(email_secret_key=email_secret_key) if email_results['email_address_object_found']: email_address_object = email_results['email_address_object'] email_address_found = True @@ -1221,7 +1323,7 @@ def voter_email_address_verify_for_api(voter_device_id, email_secret_key): # vo # If we verify it but don't use it to sign in, don't set voter_ownership_saved # (which invalidates email_secret_key below) else: - email_results = email_manager.retrieve_email_address_object_from_secret_key(email_secret_key) + email_results = email_manager.retrieve_email_address_object_from_secret_key(email_secret_key=email_secret_key) if email_results['email_address_object_found']: status += "EMAIL_ADDRESS_FOUND_FROM_RETRIEVE " email_address_object = email_results['email_address_object'] @@ -1316,17 +1418,18 @@ def voter_email_address_verify_for_api(voter_device_id, email_secret_key): # vo return json_data -def voter_email_address_save_for_api(voter_device_id='', - text_for_email_address='', - incoming_email_we_vote_id='', - send_link_to_sign_in=False, - send_sign_in_code_email=False, - resend_verification_email=False, - resend_verification_code_email=False, - make_primary_email=False, - delete_email=False, - is_cordova=False, - web_app_root_url=''): +def voter_email_address_save_for_api( + voter_device_id='', + text_for_email_address='', + incoming_email_we_vote_id='', + send_link_to_sign_in=False, + send_sign_in_code_email=False, + resend_verification_email=False, + resend_verification_code_email=False, + make_primary_email=False, + delete_email=False, + is_cordova=False, + web_app_root_url=''): """ voterEmailAddressSave :param voter_device_id: @@ -1470,6 +1573,7 @@ def voter_email_address_save_for_api(voter_device_id='', email_manager = EmailManager() email_address_already_owned_by_this_voter = False email_address_already_owned_by_other_voter = False + recipient_email_subscription_secret_key = '' verified_email_address_object = EmailAddress() verified_email_address_we_vote_id = "" email_address_list = [] @@ -1480,9 +1584,12 @@ def voter_email_address_save_for_api(voter_device_id='', if find_verified_email_results['email_address_object_found']: verified_email_address_object = find_verified_email_results['email_address_object'] verified_email_address_we_vote_id = verified_email_address_object.we_vote_id + # The only person who will see this is someone who has access to this verified_email_address + recipient_email_subscription_secret_key = verified_email_address_object.subscription_secret_key + if send_sign_in_code_email: + sign_in_code_email_already_valid = True if verified_email_address_object.voter_we_vote_id != voter_we_vote_id: email_address_already_owned_by_other_voter = True - if email_address_already_owned_by_other_voter: status += "EMAIL_ALREADY_OWNED " if send_link_to_sign_in or send_sign_in_code_email: @@ -1564,6 +1671,8 @@ def voter_email_address_save_for_api(voter_device_id='', excess_email_objects.append(ownership_not_verified_email_object) if send_sign_in_code_email: sign_in_code_email_already_valid = True + if not recipient_email_subscription_secret_key: + recipient_email_subscription_secret_key = ownership_verified_email_object.subscription_secret_key elif ownership_not_verified_email_object is not None: status += "UNVERIFIED_EMAIL_FOUND " filtered_email_address_list.append(ownership_not_verified_email_object) @@ -1666,7 +1775,7 @@ def voter_email_address_save_for_api(voter_device_id='', status += "SAVED_EMAIL_ADDRESS_AS_NEW_PRIMARY " success = True except Exception as e: - status += "UNABLE_TO_SAVE_EMAIL_ADDRESS_AS_NEW_PRIMARY " + status += "UNABLE_TO_SAVE_EMAIL_ADDRESS_AS_NEW_PRIMARY: " + str(e) + " " remove_cached_results = \ voter_manager.remove_voter_cached_email_entries_from_email_address_object( email_address_object_for_promotion) @@ -1748,7 +1857,6 @@ def voter_email_address_save_for_api(voter_device_id='', status += "LOOKING_AT_EMAIL_WITHOUT_WIPING_OUT_VOTER_PRIMARY " send_verification_email = False - recipient_email_subscription_secret_key = '' if email_address_deleted: # We cannot proceed with this email address, since it was just marked deleted pass @@ -1775,12 +1883,14 @@ def voter_email_address_save_for_api(voter_device_id='', else: recipient_email_address_secret_key = \ email_manager.update_email_address_with_new_secret_key(email_address_we_vote_id) - if positive_value_exists(new_email_address_object.subscription_secret_key): - recipient_email_subscription_secret_key = new_email_address_object.subscription_secret_key - else: - recipient_email_subscription_secret_key = \ - email_manager.update_email_address_with_new_subscription_secret_key( - email_we_vote_id=email_address_we_vote_id) + # If email_address_already_owned_by_other_voter, use the existing subscription_secret_key + if not email_address_already_owned_by_other_voter: + if positive_value_exists(new_email_address_object.subscription_secret_key): + recipient_email_subscription_secret_key = new_email_address_object.subscription_secret_key + else: + recipient_email_subscription_secret_key = \ + email_manager.update_email_address_with_new_subscription_secret_key( + email_we_vote_id=email_address_we_vote_id) email_address_created = True email_address_found = True success = True @@ -1810,7 +1920,7 @@ def voter_email_address_save_for_api(voter_device_id='', if email_scheduled_saved: link_to_sign_in_email_sent = True success = True - elif send_sign_in_code_email and not sign_in_code_email_already_valid: + elif send_sign_in_code_email: # Run the code to send email with sign in verification code (6 digit) email_address_we_vote_id = email_address_we_vote_id if positive_value_exists(email_address_we_vote_id) \ else incoming_email_we_vote_id @@ -1846,17 +1956,18 @@ def voter_email_address_save_for_api(voter_device_id='', else: status += "VOTER_DEVICE_LINK_NOT_UPDATED_WITH_EMAIL_SECRET_KEY " - email_subscription_secret_key = '' - results = email_manager.retrieve_email_address_object( - email_address_object_we_vote_id=email_address_we_vote_id) - if results['email_address_object_found']: - recipient_email_address_object = results['email_address_object'] - if positive_value_exists(recipient_email_address_object.subscription_secret_key): - email_subscription_secret_key = recipient_email_address_object.subscription_secret_key - else: - email_subscription_secret_key = \ - email_manager.update_email_address_with_new_subscription_secret_key( - email_we_vote_id=email_address_we_vote_id) + if not sign_in_code_email_already_valid: + recipient_email_subscription_secret_key = '' + results = email_manager.retrieve_email_address_object( + email_address_object_we_vote_id=email_address_we_vote_id) + if results['email_address_object_found']: + recipient_email_address_object = results['email_address_object'] + if positive_value_exists(recipient_email_address_object.subscription_secret_key): + recipient_email_subscription_secret_key = recipient_email_address_object.subscription_secret_key + else: + recipient_email_subscription_secret_key = \ + email_manager.update_email_address_with_new_subscription_secret_key( + email_we_vote_id=email_address_we_vote_id) status += 'ABOUT_TO_SEND_SIGN_IN_CODE ' link_send_results = schedule_sign_in_code_email( @@ -1865,7 +1976,7 @@ def voter_email_address_save_for_api(voter_device_id='', recipient_email_we_vote_id=email_address_we_vote_id, recipient_voter_email=text_for_email_address, secret_numerical_code=secret_code, - recipient_email_subscription_secret_key=email_subscription_secret_key, + recipient_email_subscription_secret_key=recipient_email_subscription_secret_key, web_app_root_url=web_app_root_url) status += link_send_results['status'] email_scheduled_saved = link_send_results['email_scheduled_saved'] @@ -1930,3 +2041,322 @@ def voter_email_address_save_for_api(voter_device_id='', 'secret_code_system_locked_for_this_voter_device_id': secret_code_system_locked_for_this_voter_device_id, } return json_data + + +def voter_email_address_send_sign_in_code_email_for_api( # voterEmailAddressSave + voter_device_id='', + text_for_email_address='', + web_app_root_url=''): + """ + + :param voter_device_id: + :param text_for_email_address: + :param web_app_root_url: + :return: + """ + email_address_we_vote_id = "" + email_address_created = False + email_address_deleted = False + email_address_not_valid = False + verification_email_sent = False + link_to_sign_in_email_sent = False + sign_in_code_email_sent = False + email_address_list_found = False + status = "VOTER_EMAIL_ADDRESS_SAVE-START " + error_results = { + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'text_for_email_address': text_for_email_address, + 'email_address_we_vote_id': '', + 'email_address_created': False, + 'email_address_deleted': False, + 'email_address_not_valid': False, + 'verification_email_sent': False, + 'link_to_sign_in_email_sent': False, + 'sign_in_code_email_sent': False, + 'email_address_already_owned_by_other_voter': False, + 'email_address_already_owned_by_this_voter': False, + 'email_address_found': False, + 'email_address_list_found': False, + 'email_address_list': [], + 'secret_code_system_locked_for_this_voter_device_id': False, + } + + # If a voter_device_id is passed in that isn't valid, we want to throw an error + device_id_results = is_voter_device_id_valid(voter_device_id) + if not device_id_results['success']: + status += device_id_results['status'] + " VOTER_DEVICE_ID_NOT_VALID " + error_results['status'] = status + return error_results + + # Is the text_for_email_address a valid email address? + if positive_value_exists(text_for_email_address): + if not validate_email(text_for_email_address): + status += "VOTER_EMAIL_ADDRESS_SAVE_MISSING_VALID_EMAIL " + error_results['status'] = status + return error_results + else: + status += "VOTER_EMAIL_ADDRESS_SAVE_MISSING_EMAIL " + error_results['status'] = status + return error_results + + voter_manager = VoterManager() + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_id = voter_results['voter_id'] + if not positive_value_exists(voter_id): + status += "VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID " + error_results['status'] = status + return error_results + voter = voter_results['voter'] + voter_we_vote_id = voter.we_vote_id + + email_manager = EmailManager() + email_address_already_owned_by_this_voter = False + email_address_already_owned_by_other_voter = False + email_address_found = False + + # Independent of email verification, we need a consistent recipient_email_subscription_secret_key + # If there is an email_with_ownership_verified, we need to use that above all others, + # and if not, we can generate it with a new email created further below. + recipient_email_subscription_secret_key = '' + # For verification purposes (stored in voter_device_link), this might come from temporary email address + recipient_email_address_secret_key = '' + # Is this email already verified by any account? + find_verified_email_results = email_manager.retrieve_primary_email_with_ownership_verified( + voter_we_vote_id='', + normalized_email_address=text_for_email_address) + if not find_verified_email_results['success']: + status += "PROBLEM_RETRIEVING_EMAIL: " + find_verified_email_results['status'] + " " + error_results['status'] = status + return error_results + elif find_verified_email_results['email_address_object_found']: + verified_email_address_object = find_verified_email_results['email_address_object'] + email_address_we_vote_id = verified_email_address_object.we_vote_id + # The only person who will see subscription_secret_key is someone who has access to this verified_email_address + recipient_email_subscription_secret_key = verified_email_address_object.subscription_secret_key + if verified_email_address_object.voter_we_vote_id != voter_we_vote_id: + email_address_already_owned_by_other_voter = True + # The email.secret_key isn't shown to voter + if positive_value_exists(verified_email_address_object.secret_key): + recipient_email_address_secret_key = verified_email_address_object.secret_key + status += "EXISTING_SECRET_KEY_FOUND " + else: + recipient_email_address_secret_key = \ + email_manager.update_email_address_with_new_secret_key(email_address_we_vote_id) + if positive_value_exists(recipient_email_address_secret_key): + status += "NEW_SECRET_KEY_GENERATED " + else: + status += "NEW_SECRET_KEY_COULD_NOT_BE_GENERATED " + email_address_created = False + email_address_found = True + + # From here on down, if email_address_we_vote_id is empty, we know we need to retrieve an + # email_address_object specific to this voter, or create one + if not positive_value_exists(email_address_we_vote_id): + # Look to see if there is an EmailAddress entry for the incoming text_for_email_address for this voter + email_results = email_manager.retrieve_email_address_object( + normalized_email_address=text_for_email_address, + voter_we_vote_id=voter_we_vote_id) + if email_results['email_address_object_found']: + voter_email_address_object = email_results['email_address_object'] + email_address_we_vote_id = voter_email_address_object.we_vote_id + if not recipient_email_subscription_secret_key: + recipient_email_subscription_secret_key = voter_email_address_object.subscription_secret_key + email_address_created = False + email_address_found = True + elif email_results['email_address_list_found']: + # This email was used by more than one person + email_address_list = email_results['email_address_list'] + + # Clean up our email list + # 1) Remove duplicates + excess_email_objects = [] + ownership_verified_email_object = None + ownership_verified_emails = [] + ownership_not_verified_email_object = None + ownership_not_verified_emails = [] + for email_address_object in email_address_list: + if email_address_object.email_ownership_is_verified: + if email_address_object.normalized_email_address not in ownership_verified_emails: + ownership_verified_email_object = email_address_object + ownership_verified_emails.append(email_address_object.normalized_email_address) + else: + excess_email_objects.append(email_address_object) + else: + if email_address_object.normalized_email_address not in ownership_not_verified_emails: + ownership_not_verified_email_object = email_address_object + ownership_not_verified_emails.append(email_address_object.normalized_email_address) + else: + excess_email_objects.append(email_address_object) + + if ownership_verified_email_object is not None: + email_address_we_vote_id = ownership_verified_email_object.we_vote_id + email_address_created = False + email_address_found = True + status += "VERIFIED_EMAIL_FOUND " + excess_email_objects.append(ownership_not_verified_email_object) + if not recipient_email_subscription_secret_key: + recipient_email_subscription_secret_key = ownership_verified_email_object.subscription_secret_key + # The email.secret_key isn't shown to voter + if positive_value_exists(ownership_verified_email_object.secret_key): + recipient_email_address_secret_key = ownership_verified_email_object.secret_key + status += "EXISTING_SECRET_KEY_FOUND2 " + else: + recipient_email_address_secret_key = \ + email_manager.update_email_address_with_new_secret_key(email_address_we_vote_id) + if positive_value_exists(recipient_email_address_secret_key): + status += "NEW_SECRET_KEY_GENERATED2 " + else: + status += "NEW_SECRET_KEY_COULD_NOT_BE_GENERATED2 " + elif ownership_not_verified_email_object is not None: + email_address_we_vote_id = ownership_not_verified_email_object.we_vote_id + email_address_created = False + email_address_found = True + status += "UNVERIFIED_EMAIL_FOUND " + if not recipient_email_subscription_secret_key: + recipient_email_subscription_secret_key = \ + ownership_not_verified_email_object.subscription_secret_key + # The email.secret_key isn't shown to voter + if positive_value_exists(ownership_not_verified_email_object.secret_key): + recipient_email_address_secret_key = ownership_not_verified_email_object.secret_key + status += "EXISTING_SECRET_KEY_FOUND3 " + else: + recipient_email_address_secret_key = \ + email_manager.update_email_address_with_new_secret_key(email_address_we_vote_id) + if positive_value_exists(recipient_email_address_secret_key): + status += "NEW_SECRET_KEY_GENERATED3 " + else: + status += "NEW_SECRET_KEY_COULD_NOT_BE_GENERATED3 " + + # Delete the duplicates from the database + for email_address_object in excess_email_objects: + try: + email_address_object.delete() + except Exception as e: + status += "CANNOT_DELETE_EXCESS_EMAIL: " + str(e) + " " + + if not positive_value_exists(email_address_we_vote_id): + # Save the new email address + status += "CREATE_NEW_EMAIL_ADDRESS " + email_ownership_is_verified = False + email_save_results = email_manager.create_email_address( + normalized_email_address=text_for_email_address, + voter_we_vote_id=voter_we_vote_id, + email_ownership_is_verified=email_ownership_is_verified) + status += email_save_results['status'] + if email_save_results['email_address_object_saved']: + new_email_address_object = email_save_results['email_address_object'] + email_address_we_vote_id = new_email_address_object.we_vote_id + if positive_value_exists(new_email_address_object.subscription_secret_key): + recipient_email_subscription_secret_key = new_email_address_object.subscription_secret_key + else: + recipient_email_subscription_secret_key = \ + email_manager.update_email_address_with_new_subscription_secret_key( + email_we_vote_id=email_address_we_vote_id) + # The email.secret_key isn't shown to voter + if positive_value_exists(new_email_address_object.secret_key): + recipient_email_address_secret_key = new_email_address_object.secret_key + status += "EXISTING_SECRET_KEY_FOUND4 " + else: + recipient_email_address_secret_key = \ + email_manager.update_email_address_with_new_secret_key(email_address_we_vote_id) + if positive_value_exists(recipient_email_address_secret_key): + status += "NEW_SECRET_KEY_GENERATED4 " + else: + status += "NEW_SECRET_KEY_COULD_NOT_BE_GENERATED4 " + email_address_created = True + email_address_found = True + status += email_save_results['status'] + else: + status += "UNABLE_TO_CREATE_EMAIL_ADDRESS " + error_results['status'] = status + return error_results + + voter_device_link_manager = VoterDeviceLinkManager() + # Run the code to send email with sign in verification code (6 digit) + status += "ABOUT_TO_SEND_SIGN_IN_CODE_EMAIL: " + str(email_address_we_vote_id) + " " + # We need to link a randomly generated 6 digit code to this voter_device_id + results = voter_device_link_manager.retrieve_voter_secret_code_up_to_date(voter_device_id) + secret_code = results['secret_code'] + secret_code_system_locked_for_this_voter_device_id = \ + results['secret_code_system_locked_for_this_voter_device_id'] + + if positive_value_exists(secret_code_system_locked_for_this_voter_device_id): + status += "SECRET_CODE_SYSTEM_LOCKED-EMAIL_SAVE " + success = True + elif positive_value_exists(secret_code): + # And we need to store the secret_key (as opposed to the 6 digit secret code) in the voter_device_link + # so we can match this email to this session + link_results = voter_device_link_manager.retrieve_voter_device_link(voter_device_id) + if link_results['voter_device_link_found']: + voter_device_link = link_results['voter_device_link'] + update_results = voter_device_link_manager.update_voter_device_link_with_email_secret_key( + voter_device_link, recipient_email_address_secret_key) + if positive_value_exists(update_results['success']): + status += "UPDATED_VOTER_DEVICE_LINK_WITH_SECRET_KEY " + else: + status += update_results['status'] + status += "COULD_NOT_UPDATE_VOTER_DEVICE_LINK_WITH_SECRET_KEY " + # Wipe out existing value and save again + voter_device_link_manager.clear_secret_key(email_secret_key=recipient_email_address_secret_key) + update_results = voter_device_link_manager.update_voter_device_link_with_email_secret_key( + voter_device_link, recipient_email_address_secret_key) + if not positive_value_exists(update_results['success']): + status += update_results['status'] + else: + status += "VOTER_DEVICE_LINK_NOT_UPDATED_WITH_EMAIL_SECRET_KEY " + + status += 'ABOUT_TO_SEND_SIGN_IN_CODE ' + link_send_results = schedule_sign_in_code_email( + sender_voter_we_vote_id=voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id, + recipient_email_we_vote_id=email_address_we_vote_id, + recipient_voter_email=text_for_email_address, + secret_numerical_code=secret_code, + recipient_email_subscription_secret_key=recipient_email_subscription_secret_key, + web_app_root_url=web_app_root_url) + status += link_send_results['status'] + email_scheduled_saved = link_send_results['email_scheduled_saved'] + if email_scheduled_saved: + status += "EMAIL_CODE_SCHEDULED " + sign_in_code_email_sent = True + success = True + else: + status += 'SCHEDULE_SIGN_IN_CODE_EMAIL_FAILED ' + success = False + else: + status += results['status'] + status += 'RETRIEVE_VOTER_SECRET_CODE_UP_TO_DATE_FAILED ' + success = False + + # Now that the save is complete, retrieve the updated list + email_address_list_augmented = [] + email_results = email_manager.retrieve_voter_email_address_list(voter_we_vote_id) + if email_results['email_address_list_found']: + email_address_list_found = True + email_address_list = email_results['email_address_list'] + augment_results = augment_email_address_list(email_address_list, voter) + email_address_list_augmented = augment_results['email_address_list'] + status += augment_results['status'] + + json_data = { + 'status': status, + 'success': success, + 'voter_device_id': voter_device_id, + 'text_for_email_address': text_for_email_address, + 'email_address_we_vote_id': email_address_we_vote_id, + 'email_address_already_owned_by_other_voter': email_address_already_owned_by_other_voter, + 'email_address_already_owned_by_this_voter': email_address_already_owned_by_this_voter, + 'email_address_found': email_address_found, + 'email_address_list_found': email_address_list_found, + 'email_address_list': email_address_list_augmented, + 'email_address_created': email_address_created, + 'email_address_deleted': email_address_deleted, + 'email_address_not_valid': email_address_not_valid, + 'verification_email_sent': verification_email_sent, + 'link_to_sign_in_email_sent': link_to_sign_in_email_sent, + 'sign_in_code_email_sent': sign_in_code_email_sent, + 'secret_code_system_locked_for_this_voter_device_id': secret_code_system_locked_for_this_voter_device_id, + } + return json_data diff --git a/email_outbound/models.py b/email_outbound/models.py index b72db0cf5..76e467bcd 100644 --- a/email_outbound/models.py +++ b/email_outbound/models.py @@ -3,9 +3,10 @@ # -*- coding: UTF-8 -*- from datetime import date, timedelta -from django.core.mail import EmailMultiAlternatives from django.apps import apps from django.db import models +from django.core.mail import EmailMultiAlternatives, get_connection +from config.base import get_environment_variable from wevote_functions.functions import convert_to_int, extract_email_addresses_from_string, generate_random_string, \ positive_value_exists from wevote_settings.models import fetch_next_we_vote_id_email_integer, fetch_site_unique_id_prefix @@ -57,6 +58,14 @@ (SENT, 'Message sent'), ) +SENDGRID_API_KEY = get_environment_variable("SENDGRID_API_KEY", no_exception=True) + +EMAIL_HOST = get_environment_variable("EMAIL_HOST", no_exception=True) +EMAIL_HOST_USER = get_environment_variable("EMAIL_HOST_USER", no_exception=True) +EMAIL_HOST_PASSWORD = get_environment_variable("EMAIL_HOST_PASSWORD", no_exception=True) +EMAIL_PORT = get_environment_variable("EMAIL_PORT", no_exception=True) +EMAIL_USE_TLS = get_environment_variable("EMAIL_USE_TLS", no_exception=True) + class EmailAddress(models.Model): """ @@ -123,6 +132,8 @@ class EmailOutboundDescription(models.Model): # We include this here for data monitoring and debugging recipient_voter_email = models.EmailField( verbose_name='email address for recipient', max_length=255, null=True, blank=True, unique=False) + list_unsubscribe_mailto = models.TextField(null=True, blank=True) + list_unsubscribe_url = models.TextField(null=True, blank=True) template_variables_in_json = models.TextField(null=True, blank=True) date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) @@ -147,6 +158,8 @@ class EmailScheduled(models.Model): verbose_name="we vote id for the email", max_length=255, null=True, blank=True, unique=False) recipient_voter_email = models.EmailField( verbose_name='recipient email address', max_length=255, null=True, blank=True, unique=False) + list_unsubscribe_mailto = models.TextField(null=True, blank=True) + list_unsubscribe_url = models.TextField(null=True, blank=True) send_status = models.CharField(max_length=50, choices=SEND_STATUS_CHOICES, default=TO_BE_PROCESSED) email_outbound_description_id = models.PositiveIntegerField( verbose_name="the internal id of EmailOutboundDescription", default=0, null=False) @@ -205,8 +218,12 @@ def clear_secret_key_from_email_address(self, email_secret_key): def create_email_address_for_voter(self, normalized_email_address, voter, email_ownership_is_verified=False): return self.create_email_address(normalized_email_address, voter.we_vote_id, email_ownership_is_verified) - def create_email_address(self, normalized_email_address, voter_we_vote_id='', email_ownership_is_verified=False, - make_primary_email=True): + def create_email_address( + self, + normalized_email_address='', + voter_we_vote_id='', + email_ownership_is_verified=False, + make_primary_email=True): secret_key = generate_random_string(12) status = "" normalized_email_address = str(normalized_email_address) @@ -249,10 +266,18 @@ def create_email_address(self, normalized_email_address, voter_we_vote_id='', em return results def create_email_outbound_description( - self, sender_voter_we_vote_id, sender_voter_email, sender_voter_name='', + self, + sender_voter_we_vote_id='', + sender_voter_email='', + sender_voter_name='', recipient_voter_we_vote_id='', - recipient_email_we_vote_id='', recipient_voter_email='', template_variables_in_json='', - kind_of_email_template=''): + recipient_email_we_vote_id='', + recipient_voter_email='', + template_variables_in_json='', + kind_of_email_template='', + list_unsubscribe_mailto=None, + list_unsubscribe_url=None, + ): status = "" if not positive_value_exists(kind_of_email_template): kind_of_email_template = GENERIC_EMAIL_TEMPLATE @@ -267,6 +292,8 @@ def create_email_outbound_description( recipient_voter_email=recipient_voter_email, kind_of_email_template=kind_of_email_template, template_variables_in_json=template_variables_in_json, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, ) email_outbound_description_saved = True success = True @@ -369,7 +396,7 @@ def find_and_merge_all_duplicate_emails(self, voter_we_vote_id): def merge_two_duplicate_emails(self, email_address_object1, email_address_object2): """ - We assume that the checking to see if these are duplicates has been done outside of this function. + We assume that the checking to see if these are duplicates has been done outside this function. We will keep email_address_object1 and eliminate email_address_object2. :param email_address_object1: :param email_address_object2: @@ -444,17 +471,21 @@ def parse_raw_emails_into_list(self, email_addresses_raw): success = True status = "EMAIL_MANAGER_PARSE_RAW_EMAILS" email_list = extract_email_addresses_from_string(email_addresses_raw) + at_least_one_email_found = email_list and len(email_list) > 0 results = { 'success': success, 'status': status, - 'at_least_one_email_found': True, + 'at_least_one_email_found': at_least_one_email_found, 'email_list': email_list, } return results - def retrieve_email_address_object(self, normalized_email_address='', email_address_object_we_vote_id='', - voter_we_vote_id=''): + def retrieve_email_address_object( + self, + normalized_email_address='', + email_address_object_we_vote_id='', + voter_we_vote_id=''): """ There are cases where we store multiple entries for the same normalized_email_address (prior to an email address being verified) @@ -606,7 +637,7 @@ def retrieve_email_address_object_from_secret_key(self, email_secret_key='', sub } return results - def verify_email_address_object_from_secret_key(self, email_secret_key): + def verify_email_address_object_from_secret_key(self, email_secret_key=''): """ :param email_secret_key: @@ -724,7 +755,7 @@ def retrieve_voter_email_address_list(self, voter_we_vote_id): } return results - def retrieve_primary_email_with_ownership_verified(self, voter_we_vote_id, normalized_email_address=''): + def retrieve_primary_email_with_ownership_verified(self, voter_we_vote_id='', normalized_email_address=''): status = "" email_address_list = [] email_address_list_found = False @@ -787,6 +818,20 @@ def fetch_primary_email_with_ownership_verified(self, voter_we_vote_id): return "" + def fetch_simple_voter_email_address_list(self, voter_we_vote_id): + simple_email_address_list = [] + results = self.retrieve_voter_email_address_list(voter_we_vote_id=voter_we_vote_id) + if results['email_address_list_found']: + email_address_list = results['email_address_list'] + for email_object in email_address_list: + # We don't care if the email is confirmed or not -- if the voter thinks the email is there own, + # don't show it in the contacts list + if positive_value_exists(email_object.normalized_email_address) and \ + email_object.normalized_email_address not in simple_email_address_list: + simple_email_address_list.append(email_object.normalized_email_address) + + return simple_email_address_list + def retrieve_scheduled_email_list_from_send_status(self, sender_voter_we_vote_id, send_status): status = "" scheduled_email_list = [] @@ -836,8 +881,13 @@ def update_scheduled_email_with_new_send_status(self, email_scheduled_object, se print(status) return email_scheduled_object - def schedule_email(self, email_outbound_description, subject, message_text, message_html, - send_status=TO_BE_PROCESSED): + def schedule_email( + self, + email_outbound_description=None, + subject="", + message_text="", + message_html="", + send_status=TO_BE_PROCESSED): status = '' try: email_scheduled = EmailScheduled.objects.create( @@ -852,6 +902,8 @@ def schedule_email(self, email_outbound_description, subject, message_text, mess email_outbound_description_id=email_outbound_description.id, send_status=send_status, subject=subject, + list_unsubscribe_mailto=email_outbound_description.list_unsubscribe_mailto, + list_unsubscribe_url=email_outbound_description.list_unsubscribe_url, ) email_scheduled_saved = True email_scheduled_id = email_scheduled.id @@ -862,7 +914,7 @@ def schedule_email(self, email_outbound_description, subject, message_text, mess email_scheduled = EmailScheduled() email_scheduled_id = 0 success = False - status += "ERROR_SCHEDULE_EMAIL_NOT_CREATED " + str(e) + ' ' + status += "ERROR_SCHEDULE_EMAIL_NOT_CREATED: " + str(e) + ' ' print(status) results = { @@ -898,7 +950,11 @@ def send_scheduled_email(self, email_scheduled): success = False if success: - return self.send_scheduled_email_via_sendgrid(email_scheduled) + send_via_sendgrid = True + if send_via_sendgrid: + return self.send_scheduled_email_via_sendgrid(email_scheduled) + else: + return self.send_scheduled_email_via_smtp(email_scheduled) else: status += "ERROR_DID_NOT_SEND: [" try: @@ -921,8 +977,11 @@ def send_scheduled_email_via_sendgrid(self, email_scheduled): :param email_scheduled: :return: """ + from sendgrid import SendGridAPIClient + from sendgrid.helpers.mail import Content, From, Header, Mail, MimeType, Subject, To, ReplyTo status = "" success = True + email_scheduled_sent = False sendgrid_turned_off_for_testing = False if sendgrid_turned_off_for_testing: status += "ERROR_SENDGRID_TURNED_OFF_FOR_TESTING " @@ -934,32 +993,148 @@ def send_scheduled_email_via_sendgrid(self, email_scheduled): } return results - if positive_value_exists(email_scheduled.sender_voter_name): - # TODO DALE Make system variable - system_sender_email_address = "{sender_voter_name} via We Vote " \ - "".format(sender_voter_name=email_scheduled.sender_voter_name) - else: - system_sender_email_address = "We Vote " # TODO DALE Make system variable - - mail = EmailMultiAlternatives( - subject=email_scheduled.subject, - body=email_scheduled.message_text, - from_email=system_sender_email_address, - to=[email_scheduled.recipient_voter_email], - # headers={"Reply-To": email_scheduled.sender_voter_email} - ) - # 2020-01-19 Dale commented out Reply-To header because with it, Gmail gives phishing warning - if positive_value_exists(email_scheduled.message_html): - mail.attach_alternative(email_scheduled.message_html, "text/html") + try: + message = Mail() + if positive_value_exists(email_scheduled.sender_voter_name): + message.from_email = From( + 'info@wevote.us', + "{sender_voter_name} via We Vote".format(sender_voter_name=email_scheduled.sender_voter_name)) + else: + message.from_email = From('info@wevote.us', 'We Vote') + message.reply_to = ReplyTo('info@wevote.us', 'We Vote') + message.to = To(email_scheduled.recipient_voter_email, email_scheduled.recipient_voter_email, p=0) + try: + if email_scheduled.list_unsubscribe_mailto or email_scheduled.list_unsubscribe_url: + list_unsubscribe_text = '' + if email_scheduled.list_unsubscribe_mailto: + list_unsubscribe_text += \ + "" \ + "".format(list_unsubscribe_mailto=email_scheduled.list_unsubscribe_mailto) + if email_scheduled.list_unsubscribe_url: + list_unsubscribe_text += ", " + if email_scheduled.list_unsubscribe_url: + list_unsubscribe_text += \ + "<{list_unsubscribe_url}>" \ + "".format(list_unsubscribe_url=email_scheduled.list_unsubscribe_url) + message.add_header( + Header(key="List-Unsubscribe", value=list_unsubscribe_text) + ) + if email_scheduled.list_unsubscribe_url: + message.add_header( + Header(key="List-Unsubscribe-Post", value="List-Unsubscribe=One-Click") + ) + except Exception as e: + status += "SEND_SCHEDULED_ADD_HEADER_ERROR: " + str(e) + " " + print(status) + message.subject = Subject(email_scheduled.subject) + message.content = Content( + MimeType.text, + email_scheduled.message_text) + message.content = Content( + MimeType.html, + email_scheduled.message_html) + try: + sendgrid_client = SendGridAPIClient(SENDGRID_API_KEY) + response = sendgrid_client.send(message) + # print(response.status_code) + # print(response.body) + # print(response.headers) + status += "SENDING_VIA_SENDGRID " + email_scheduled_sent = True + except Exception as e: + status += "ERROR_COULD_NOT_SEND_VIA_SENDGRID: " + str(e) + ' ' + print(status) + email_scheduled_sent = False + except Exception as e: + status += "ERROR_COULD_NOT_BE_PREPARED_FOR_SENDGRID: " + str(e) + ' ' + print(status) + + results = { + 'success': success, + 'status': status, + 'email_scheduled_sent': email_scheduled_sent, + } + return results + + def send_scheduled_email_via_smtp(self, email_scheduled): + """ + Send a single scheduled email but using the SMTP settings in environment_variables + :param email_scheduled: + :return: + """ + status = "" + success = True + email_scheduled_sent = False + smtp_turned_off_for_testing = False + if smtp_turned_off_for_testing: + status += "ERROR_SMTP_TURNED_OFF_FOR_TESTING " + print(status) + results = { + 'success': success, + 'status': status, + 'email_scheduled_sent': False, + } + return results try: - mail.send() - status += "SENDING_VIA_SENDGRID " - email_scheduled_sent = True + # Prepare headers_dict + headers_dict = {} + try: + if email_scheduled.list_unsubscribe_mailto or email_scheduled.list_unsubscribe_url: + list_unsubscribe_text = '' + if email_scheduled.list_unsubscribe_mailto: + list_unsubscribe_text += \ + "" \ + "".format(list_unsubscribe_mailto=email_scheduled.list_unsubscribe_mailto) + if email_scheduled.list_unsubscribe_url: + list_unsubscribe_text += ", " + if email_scheduled.list_unsubscribe_url: + list_unsubscribe_text += \ + "<{list_unsubscribe_url}>" \ + "".format(list_unsubscribe_url=email_scheduled.list_unsubscribe_url) + headers_dict["List-Unsubscribe"] = list_unsubscribe_text + if email_scheduled.list_unsubscribe_url: + headers_dict["List-Unsubscribe-Post"] = "List-Unsubscribe=One-Click" + except Exception as e: + status += "SEND_SCHEDULED_ADD_HEADER_ERROR: " + str(e) + " " + print(status) + if positive_value_exists(email_scheduled.sender_voter_name): + from_email = "{sender_voter_name} via We Vote " \ + "".format(email_address='info@wevote.us', + sender_voter_name=email_scheduled.sender_voter_name) + else: + from_email = "We Vote " \ + "".format(email_address='info@wevote.us') + # For some reason the default Emailbackend doesn't have access to environment_variables.json directly + connection = get_connection( + username=EMAIL_HOST_USER, + password=EMAIL_HOST_PASSWORD, + host=EMAIL_HOST, + port=EMAIL_PORT) + connection.open() + message = EmailMultiAlternatives( + subject=email_scheduled.subject, + body=email_scheduled.message_text, + from_email=from_email, + to=[email_scheduled.recipient_voter_email], + connection=connection, + reply_to=['We Vote '], + headers=headers_dict, + ) + message.attach_alternative(email_scheduled.message_html, "text/html") + + try: + message.send(fail_silently=False) + connection.close() + status += "SENDING_VIA_SMTP " + email_scheduled_sent = True + except Exception as e: + status += "ERROR_COULD_NOT_SEND_VIA_SMTP: " + str(e) + ' ' + print(status) + email_scheduled_sent = False except Exception as e: - status += "ERROR_COULD_NOT_SEND_VIA_SENDGRID: " + str(e) + ' ' + status += "ERROR_COULD_NOT_BE_PREPARED_FOR_SMTP: " + str(e) + ' ' print(status) - email_scheduled_sent = False results = { 'success': success, @@ -1061,18 +1236,28 @@ def update_email_address_with_new_secret_key(self, email_we_vote_id): else: return "" - def update_email_address_with_new_subscription_secret_key(self, email_we_vote_id): + def update_email_address_with_new_subscription_secret_key(self, email_we_vote_id='', force_change=False): + """ + The subscription_secret_key is used to let the voter unsubscribe without being signed in, + so it shouldn't change. + :param email_we_vote_id: + :param force_change: + :return: + """ results = self.retrieve_email_address_object('', email_we_vote_id) if results['email_address_object_found']: email_address_object = results['email_address_object'] - try: - email_address_object.subscription_secret_key = generate_random_string(48) - email_address_object.save() + if not positive_value_exists(email_address_object.subscription_secret_key) or force_change: + try: + email_address_object.subscription_secret_key = generate_random_string(48) + email_address_object.save() + return email_address_object.subscription_secret_key + except Exception as e: + status = "ERROR_UPDATE_EMAIL_ADDRESS_WITH_NEW_SUBSCRIPTION_SECRET_KEY: " + str(e) + " " + print(status) + return "" + else: return email_address_object.subscription_secret_key - except Exception as e: - status = "ERROR_UPDATE_EMAIL_ADDRESS_WITH_NEW_SUBSCRIPTION_SECRET_KEY: " + str(e) + " " - print(status) - return "" else: return "" diff --git a/follow/controllers.py b/follow/controllers.py index e3b1ef0c6..1343728f0 100644 --- a/follow/controllers.py +++ b/follow/controllers.py @@ -227,7 +227,7 @@ def duplicate_follow_issue_entries_to_another_voter(from_voter_we_vote_id, to_vo return results -def move_follow_entries_to_another_voter(from_voter_id, to_voter_id, to_voter_we_vote_id): +def move_follow_entries_to_another_voter(from_voter_id=0, to_voter_id=0, to_voter_we_vote_id=''): status = '' success = False follow_entries_moved = 0 @@ -260,23 +260,23 @@ def move_follow_entries_to_another_voter(from_voter_id, to_voter_id, to_voter_we return results follow_organization_list = FollowOrganizationList() - follow_organization_manager = FollowOrganizationManager() - from_follow_list = follow_organization_list.retrieve_follow_organization_by_voter_id(from_voter_id) - - for from_follow_entry in from_follow_list: - # See if the "to_voter" already has an entry for this organization - existing_entry_results = follow_organization_manager.retrieve_follow_organization( - 0, to_voter_id, from_follow_entry.organization_id, from_follow_entry.organization_we_vote_id) - if not existing_entry_results['follow_organization_found']: - # Change the voter_id and voter_we_vote_id - try: - from_follow_entry.voter_id = to_voter_id - # We don't currently store follow entries by we_vote_id - # from_follow_entry.voter_we_vote_id = to_voter_we_vote_id - from_follow_entry.save() - follow_entries_moved += 1 - except Exception as e: - follow_entries_not_moved += 1 + organization_we_vote_ids_followed = \ + follow_organization_list.retrieve_follow_organization_by_voter_id_simple_id_array( + voter_id=to_voter_id, + return_we_vote_id=True, + ) + + move_results = follow_organization_list.move_follow_organization_from_voter_id_to_new_voter_id( + from_voter_id=from_voter_id, + to_voter_id=to_voter_id, + exclude_organization_we_vote_id_list=organization_we_vote_ids_followed, + ) + follow_entries_moved = move_results['number_moved'] + + if move_results['success']: + # Finally, delete remaining FollowOrganization entries for from_voter_id + delete_results = follow_organization_list.delete_follow_organization_list_for_voter_id(voter_id=from_voter_id) + follow_entries_not_moved = delete_results['number_deleted'] results = { 'status': status, diff --git a/follow/models.py b/follow/models.py index 37e3d3633..2e681aac4 100644 --- a/follow/models.py +++ b/follow/models.py @@ -1248,7 +1248,6 @@ def retrieve_follow_organization_by_voter_id(self, voter_id, auto_followed_from_ read_only=False): # Retrieve a list of follow_organization entries for this voter follow_organization_list_found = False - following_status = FOLLOWING follow_organization_list = {} try: # Should not default to 'readonly' since we sometimes save the results of this call @@ -1257,7 +1256,7 @@ def retrieve_follow_organization_by_voter_id(self, voter_id, auto_followed_from_ else: follow_organization_list = FollowOrganization.objects.all() follow_organization_list = follow_organization_list.filter(voter_id=voter_id) - follow_organization_list = follow_organization_list.filter(following_status=following_status) + follow_organization_list = follow_organization_list.filter(following_status=FOLLOWING) if auto_followed_from_twitter_suggestion: follow_organization_list = follow_organization_list.filter( auto_followed_from_twitter_suggestion=auto_followed_from_twitter_suggestion) @@ -1272,6 +1271,58 @@ def retrieve_follow_organization_by_voter_id(self, voter_id, auto_followed_from_ follow_organization_list = {} return follow_organization_list + def delete_follow_organization_list_for_voter_id( + self, + voter_id=0, + ): + number_deleted = 0 + success = True + status = '' + try: + queryset = FollowOrganization.objects.all() + queryset = queryset.filter(voter_id=voter_id) + number_deleted = queryset.delete() + status += "DELETED_FOLLOW_ORGANIZATION: " + str(number_deleted) + " " + except Exception as e: + success = False + status += "FAILED_TO_DELETE_FOLLOW_ORGANIZATION: " + str(e) + " " + + results = { + 'success': success, + 'status': status, + 'number_deleted': number_deleted, + } + return results + + def move_follow_organization_from_voter_id_to_new_voter_id( + self, + from_voter_id=0, + to_voter_id=0, + exclude_organization_we_vote_id_list=[], + ): + success = True + status = '' + try: + queryset = FollowOrganization.objects.all() + queryset = queryset.filter(voter_id=from_voter_id) + if len(exclude_organization_we_vote_id_list): + queryset = queryset.exclude(organization_we_vote_id__in=exclude_organization_we_vote_id_list) + number_moved = queryset.update( + voter_id=to_voter_id, + ) + status += "UPDATED_FOLLOW_ORGANIZATION: " + str(number_moved) + " " + except Exception as e: + number_moved = 0 + success = False + status += "FAILED_TO_UPDATE_FOLLOW_ORGANIZATION: " + str(e) + " " + + results = { + 'success': success, + 'status': status, + 'number_moved': number_moved, + } + return results + def retrieve_follow_organization_by_own_organization_we_vote_id(self, organization_we_vote_id, auto_followed_from_twitter_suggestion=False): # Retrieve a list of followed organizations entries by voter_linked_organization_we_vote_id for voter guides @@ -1320,36 +1371,27 @@ def retrieve_ignore_organization_by_voter_id(self, voter_id, read_only=False): follow_organization_list = {} return follow_organization_list - def retrieve_follow_organization_by_voter_id_simple_id_array(self, voter_id, return_we_vote_id=False, - auto_followed_from_twitter_suggestion=False, - read_only=False): - follow_organization_list_manager = FollowOrganizationList() - follow_organization_list = \ - follow_organization_list_manager.retrieve_follow_organization_by_voter_id( - voter_id, auto_followed_from_twitter_suggestion, read_only=read_only) - follow_organization_list_simple_array = [] - if len(follow_organization_list): - voter_manager = VoterManager() - voter_linked_organization_we_vote_id = \ - voter_manager.fetch_linked_organization_we_vote_id_from_local_id(voter_id) - for follow_organization in follow_organization_list: - if not read_only: - # Heal the data by making sure the voter's linked_organization_we_vote_id exists and is accurate - if positive_value_exists(voter_linked_organization_we_vote_id) \ - and voter_linked_organization_we_vote_id != \ - follow_organization.voter_linked_organization_we_vote_id: - try: - follow_organization.voter_linked_organization_we_vote_id = \ - voter_linked_organization_we_vote_id - follow_organization.save() - except Exception as e: - status = 'FAILED_TO_UPDATE_FOLLOW_ISSUE-voter_id ' + str(voter_id) - handle_record_not_saved_exception(e, logger=logger, exception_message_optional=status) + def retrieve_follow_organization_by_voter_id_simple_id_array( + self, + voter_id=0, + return_we_vote_id=False, + auto_followed_from_twitter_suggestion=False): + + try: + queryset = FollowOrganization.objects.using('readonly').all() + queryset = queryset.filter(voter_id=voter_id) + queryset = queryset.filter(following_status=FOLLOWING) + if auto_followed_from_twitter_suggestion: + queryset = queryset.filter( + auto_followed_from_twitter_suggestion=auto_followed_from_twitter_suggestion) + if positive_value_exists(return_we_vote_id): + queryset = queryset.values_list('organization_we_vote_id', flat=True).distinct() + else: + queryset = queryset.values_list('organization_id', flat=True).distinct() + follow_organization_list_simple_array = list(queryset) + except Exception as e: + follow_organization_list_simple_array = [] - if return_we_vote_id: - follow_organization_list_simple_array.append(follow_organization.organization_we_vote_id) - else: - follow_organization_list_simple_array.append(follow_organization.organization_id) return follow_organization_list_simple_array def retrieve_followed_organization_by_organization_we_vote_id_simple_id_array( @@ -1419,18 +1461,20 @@ def retrieve_followers_organization_by_organization_we_vote_id_simple_id_array( return followers_organization_list_simple_array def retrieve_ignore_organization_by_voter_id_simple_id_array( - self, voter_id, return_we_vote_id=False, read_only=False): - follow_organization_list_manager = FollowOrganizationList() - ignore_organization_list = \ - follow_organization_list_manager.retrieve_ignore_organization_by_voter_id(voter_id, read_only=read_only) - ignore_organization_list_simple_array = [] - if len(ignore_organization_list): - for ignore_organization in ignore_organization_list: - if return_we_vote_id: - ignore_organization_list_simple_array.append(ignore_organization.organization_we_vote_id) - else: - ignore_organization_list_simple_array.append(ignore_organization.organization_id) - return ignore_organization_list_simple_array + self, voter_id, return_we_vote_id=False): + try: + queryset = FollowOrganization.objects.using('readonly').all() + queryset = queryset.filter(voter_id=voter_id) + queryset = queryset.filter(following_status=FOLLOW_IGNORE) + if positive_value_exists(return_we_vote_id): + queryset = queryset.values_list('organization_we_vote_id', flat=True).distinct() + else: + queryset = queryset.values_list('organization_id', flat=True).distinct() + follow_organization_list_simple_array = list(queryset) + except Exception as e: + follow_organization_list_simple_array = [] + + return follow_organization_list_simple_array def retrieve_follow_organization_by_organization_id(self, organization_id): # Retrieve a list of follow_organization entries for this organization diff --git a/friend/controllers.py b/friend/controllers.py index 566a0b16c..547073ccb 100644 --- a/friend/controllers.py +++ b/friend/controllers.py @@ -1,15 +1,17 @@ # friend/controllers.py # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- -from .models import ACCEPTED, FriendInvitationVoterLink, FriendManager, CURRENT_FRIENDS, \ + +from django.db.models import F, Q +from django.utils.timezone import localtime, now +from .models import ACCEPTED, CurrentFriend, FriendInvitationVoterLink, FriendManager, CURRENT_FRIENDS, \ DELETE_INVITATION_EMAIL_SENT_BY_ME, DELETE_INVITATION_VOTER_SENT_BY_ME, FRIEND_INVITATIONS_PROCESSED, \ FRIEND_INVITATIONS_SENT_BY_ME, FRIEND_INVITATIONS_SENT_TO_ME, FRIEND_INVITATIONS_WAITING_FOR_VERIFICATION, \ - IGNORE_SUGGESTION, SUGGESTED_FRIEND_LIST, \ - UNFRIEND_CURRENT_FRIEND + IGNORE_SUGGESTION, MutualFriend, SuggestedFriend, SUGGESTED_FRIEND_LIST, UNFRIEND_CURRENT_FRIEND from config.base import get_environment_variable from email_outbound.controllers import schedule_email_with_email_outbound_description, schedule_verification_email from email_outbound.models import EmailAddress, EmailManager, FRIEND_ACCEPTED_INVITATION_TEMPLATE, \ - FRIEND_INVITATION_TEMPLATE, MESSAGE_TO_FRIEND_TEMPLATE, TO_BE_PROCESSED, WAITING_FOR_VERIFICATION + FRIEND_INVITATION_TEMPLATE, MESSAGE_TO_FRIEND_TEMPLATE, SENT, TO_BE_PROCESSED, WAITING_FOR_VERIFICATION from follow.models import FollowIssueList from import_export_facebook.models import FacebookManager import json @@ -126,7 +128,7 @@ def delete_friends_for_voter(voter_to_delete_we_vote_id): return results friend_manager = FriendManager() - from_friend_results = friend_manager.retrieve_current_friends(voter_to_delete_we_vote_id, read_only=False) + from_friend_results = friend_manager.retrieve_current_friend_list(voter_to_delete_we_vote_id, read_only=False) from_friend_list = from_friend_results['current_friend_list'] for from_friend_entry in from_friend_list: @@ -202,8 +204,11 @@ def fetch_friend_invitation_recipient_voter_we_vote_id(friend_invitation): return '' -def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_we_vote_id, invitation_message='', - web_app_root_url=''): +def friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + invitation_message='', + web_app_root_url=''): """ A person has accepted a friend request, so we want to email the original_sender voter who invited the accepting_voter @@ -221,8 +226,8 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ if not voter_results['voter_found']: error_results = { - 'status': "ACCEPTING_VOTER_NOT_FOUND ", - 'success': False, + 'status': "ACCEPTING_VOTER_NOT_FOUND ", + 'success': False, } return error_results @@ -231,8 +236,8 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ original_sender_voter_results = voter_manager.retrieve_voter_by_we_vote_id(original_sender_we_vote_id) if not original_sender_voter_results['voter_found']: error_results = { - 'status': "ORIGINAL_SENDER_VOTER_NOT_FOUND ", - 'success': False, + 'status': "ORIGINAL_SENDER_VOTER_NOT_FOUND ", + 'success': False, } return error_results @@ -289,6 +294,27 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ else: subject = "Friend accepted your invitation on We Vote" + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # original_sender_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/friendaccept" \ + "".format( + email_secret_key=original_sender_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendaccept/" \ + "".format( + email_secret_key=original_sender_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_FRIEND_REQUEST_RESPONSES_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='friendaccept') + template_variables_for_json = { "subject": subject, "invitation_message": invitation_message, @@ -298,11 +324,9 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ "sender_description": accepting_voter_description, "sender_network_details": accepting_voter_network_details, "recipient_name": original_sender_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": original_sender_email, "see_your_friend_list_url": web_app_root_url_verified + "/friends", - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - original_sender_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) @@ -317,7 +341,10 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ recipient_email_we_vote_id=original_sender_email_we_vote_id, recipient_voter_email=original_sender_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " if outbound_results['email_outbound_description_saved']: email_outbound_description = outbound_results['email_outbound_description'] @@ -329,18 +356,32 @@ def friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_ send_results = email_manager.send_scheduled_email(email_scheduled) email_scheduled_sent = send_results['email_scheduled_sent'] status += send_results['status'] + if email_scheduled_sent: + # If scheduled email sent successfully change their status from WAITING_FOR_VERIFICATION to SENT + send_status = SENT + try: + email_scheduled.send_status = send_status + email_scheduled.save() + except Exception as e: + status += "ERROR_FAILED_TO_UPDATE_FRIEND_ACCEPTED_SEND_STATUS: " + str(e) + ' ' + print(status) results = { - 'success': True, - 'status': status, + 'success': True, + 'status': status, } return results -def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitationByEmailSend - email_address_array, first_name_array, last_name_array, - email_addresses_raw, invitation_message, - sender_email_address, web_app_root_url=''): +def friend_invitation_by_email_send_for_api( # friendInvitationByEmailSend + voter_device_id='', + email_address_array=[], + first_name_array=[], + last_name_array=[], + email_addresses_raw='', + invitation_message='', + sender_email_address='', + web_app_root_url=''): """ :param voter_device_id: @@ -355,31 +396,37 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation """ success = True status = "" + number_of_messages_sent = 0 error_message_to_show_voter = "" sender_voter_email_address_missing = True + success_message_to_show_voter = "" results = is_voter_device_id_valid(voter_device_id) if not results['success']: error_results = { 'status': results['status'], 'success': False, - 'voter_device_id': voter_device_id, + 'error_message_to_show_voter': error_message_to_show_voter, + 'number_of_messages_sent': number_of_messages_sent, 'sender_voter_email_address_missing': sender_voter_email_address_missing, - 'error_message_to_show_voter': error_message_to_show_voter + 'success_message_to_show_voter': success_message_to_show_voter, + 'voter_device_id': voter_device_id, } return error_results voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) sender_voter_id = voter_results['voter_id'] if not positive_value_exists(sender_voter_id): status += "VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID " error_results = { 'status': status, 'success': False, - 'voter_device_id': voter_device_id, + 'error_message_to_show_voter': error_message_to_show_voter, + 'number_of_messages_sent': number_of_messages_sent, 'sender_voter_email_address_missing': sender_voter_email_address_missing, - 'error_message_to_show_voter': error_message_to_show_voter + 'success_message_to_show_voter': success_message_to_show_voter, + 'voter_device_id': voter_device_id, } return error_results @@ -446,9 +493,11 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation error_results = { 'success': False, 'status': status, - 'voter_device_id': voter_device_id, + 'error_message_to_show_voter': error_message_to_show_voter, + 'number_of_messages_sent': number_of_messages_sent, 'sender_voter_email_address_missing': sender_voter_email_address_missing, - 'error_message_to_show_voter': error_message_to_show_voter + 'success_message_to_show_voter': success_message_to_show_voter, + 'voter_device_id': voter_device_id, } return error_results @@ -486,7 +535,7 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation # We can continue. Note that we are not checking for "voter.has_email_with_verified_ownership()" pass else: - status += "VOTER_DOES_NOT_HAVE_VALID_EMAIL-CACHING_FOR_LATER " + status += "SENDER_VOTER_DOES_NOT_HAVE_VALID_EMAIL-CACHING_FOR_LATER " if not isinstance(first_name_array, (list, tuple)): first_name_array = [] @@ -494,11 +543,17 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation if not isinstance(last_name_array, (list, tuple)): last_name_array = [] + # if not positive_value_exists(invitation_message): + # invitation_message = "" if email_address_array: # Reconstruct dictionary array from lists for n in range(len(email_address_array)): - first_name = first_name_array[n] - last_name = last_name_array[n] + try: + first_name = first_name_array[n] + last_name = last_name_array[n] + except Exception as e: + first_name = '' + last_name = '' one_normalized_raw_email = email_address_array[n] # Make sure the current voter isn't already friends with owner of this email address @@ -508,18 +563,29 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation # Do not send an invitation status += is_friend_results['status'] status += "ALREADY_FRIENDS_WITH_SENDER_VOTER_EMAIL_ADDRESS_ARRAY " - error_message_to_show_voter += "You are already friends with the owner of " \ - "'{one_normalized_raw_email}'. " \ - "".format(one_normalized_raw_email=one_normalized_raw_email) + success_message_to_show_voter += \ + "You are already friends with the owner of " \ + "'{one_normalized_raw_email}'. " \ + "".format(one_normalized_raw_email=one_normalized_raw_email) continue - send_results = send_to_one_friend(voter_device_id, sender_voter, send_now, - sender_email_with_ownership_verified, - one_normalized_raw_email, first_name, last_name, invitation_message, - web_app_root_url) + send_results = send_to_one_friend( + voter_device_id=voter_device_id, + sender_voter=sender_voter, + send_now=send_now, + sender_email_with_ownership_verified=sender_email_with_ownership_verified, + one_normalized_raw_email=one_normalized_raw_email, + first_name=first_name, + last_name=last_name, + invitation_message=invitation_message, + web_app_root_url=web_app_root_url) + error_message_to_show_voter += send_results['error_message_to_show_voter'] + if send_results['success']: + number_of_messages_sent += 1 status += send_results['status'] - else: + elif positive_value_exists(email_addresses_raw): + # This branch is used for inviting single friends (e.g. Add friends from your contacts) # Break apart all the emails in email_addresses_raw input from the voter results = email_manager.parse_raw_emails_into_list(email_addresses_raw) if results['at_least_one_email_found']: @@ -539,20 +605,28 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation "".format(one_normalized_raw_email=one_normalized_raw_email) continue - send_results = send_to_one_friend(voter_device_id, sender_voter, send_now, - sender_email_with_ownership_verified, - one_normalized_raw_email, first_name, last_name, invitation_message, - web_app_root_url) + send_results = send_to_one_friend( + voter_device_id=voter_device_id, + sender_voter=sender_voter, + send_now=send_now, + sender_email_with_ownership_verified=sender_email_with_ownership_verified, + one_normalized_raw_email=one_normalized_raw_email, + first_name=first_name, + last_name=last_name, + invitation_message=invitation_message, + web_app_root_url=web_app_root_url) status += send_results['status'] else: - error_message_to_show_voter = "Please enter the email address of at least one friend." + error_message_to_show_voter = "Please enter at least one email address." status += "LIST_OF_EMAILS_NOT_RECEIVED " + results['status'] error_results = { 'status': status, 'success': False, - 'voter_device_id': voter_device_id, + 'error_message_to_show_voter': error_message_to_show_voter, + 'number_of_messages_sent': number_of_messages_sent, 'sender_voter_email_address_missing': sender_voter_email_address_missing, - 'error_message_to_show_voter': error_message_to_show_voter + 'success_message_to_show_voter': success_message_to_show_voter, + 'voter_device_id': voter_device_id, } return error_results @@ -596,16 +670,25 @@ def friend_invitation_by_email_send_for_api(voter_device_id, # friendInvitation results = { 'success': success, 'status': status, - 'voter_device_id': voter_device_id, + 'error_message_to_show_voter': error_message_to_show_voter, + 'number_of_messages_sent': number_of_messages_sent, 'sender_voter_email_address_missing': sender_voter_email_address_missing, - 'error_message_to_show_voter': error_message_to_show_voter + 'success_message_to_show_voter': success_message_to_show_voter, + 'voter_device_id': voter_device_id, } return results -def send_to_one_friend(voter_device_id, sender_voter, send_now, sender_email_with_ownership_verified, - one_normalized_raw_email, first_name, last_name, invitation_message, - web_app_root_url=''): +def send_to_one_friend( + voter_device_id='', + sender_voter=None, + send_now=False, + sender_email_with_ownership_verified='', + one_normalized_raw_email='', + first_name='', + last_name='', + invitation_message='', + web_app_root_url=''): # Starting with a raw email address, find (or create) the EmailAddress entry # and the owner (Voter) if exists status = "" @@ -719,6 +802,27 @@ def send_to_one_friend(voter_device_id, sender_voter, send_now, sender_email_wit email_manager.update_email_address_with_new_subscription_secret_key( email_we_vote_id=recipient_email_we_vote_id) + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/friendinvite" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendinvite/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_FRIEND_REQUESTS_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='friendinvite') + template_variables_for_json = { "subject": subject, "invitation_message": invitation_message, @@ -728,12 +832,10 @@ def send_to_one_friend(voter_device_id, sender_voter, send_now, sender_email_wit "sender_description": sender_description, "sender_network_details": sender_network_details, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_voter_email, "see_all_friend_requests_url": web_app_root_url_verified + "/friends", "confirm_friend_request_url": web_app_root_url_verified + "/more/network/key/" + invitation_secret_key, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) @@ -749,7 +851,10 @@ def send_to_one_friend(voter_device_id, sender_voter, send_now, sender_email_wit recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_voter_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " email_outbound_description = outbound_results['email_outbound_description'] if outbound_results['email_outbound_description_saved'] and send_now: @@ -795,7 +900,7 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key friend_we_vote_id = '' friend_organization_we_vote_id = '' invitation_message = '' - invitation_secret_key_belongs_to_this_voter = '' + invitation_secret_key_belongs_to_this_voter = True # If a voter_device_id is passed in that isn't valid, we want to throw an error device_id_results = is_voter_device_id_valid(voter_device_id) @@ -803,20 +908,20 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key status += "friendInvitationInformation-MISSING_VOTER_DEVICE_ID:" status += device_id_results['status'] json_data = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return json_data @@ -826,22 +931,22 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key 'status': status, 'success': False, 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) voter_id = voter_results['voter_id'] if not positive_value_exists(voter_id): status += "friendInvitationInformation_VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID " @@ -849,17 +954,17 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key 'status': status, 'success': False, 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results voter = voter_results['voter'] @@ -867,24 +972,26 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key friend_manager = FriendManager() friend_invitation_results = friend_manager.retrieve_friend_invitation_from_secret_key( - invitation_secret_key, for_retrieving_information=True, read_only=True) + invitation_secret_key, + for_retrieving_information=True, + read_only=True) if not friend_invitation_results['friend_invitation_found']: status += "INVITATION_NOT_FOUND_FROM_SECRET_KEY-RETRIEVING_INFO " error_results = { - 'status': status, - 'success': True, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': False, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': True, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': False, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results @@ -892,77 +999,77 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key invitation_found = True if friend_invitation_results['friend_invitation_voter_link_found']: friend_invitation_voter_link = friend_invitation_results['friend_invitation_voter_link'] - status += "FRIEND_INVITATION_VOTER_LINK_FOUND " + status += "INVITATION_INFORMATION_FRIEND_INVITATION_VOTER_LINK_FOUND " + sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id - if friend_invitation_voter_link.sender_voter_we_vote_id == voter_we_vote_id: + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED-VOTER_LINK " error_results = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': invitation_found, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': invitation_found, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results if friend_invitation_voter_link.recipient_voter_we_vote_id != voter_we_vote_id: status += "RECIPIENT_DOES_NOT_MATCH_CURRENT_VOTER-VOTER_LINK " error_results = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': invitation_found, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': invitation_found, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results - invitation_secret_key_belongs_to_this_voter = True - sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id invitation_message = friend_invitation_voter_link.invitation_message elif friend_invitation_results['friend_invitation_email_link_found']: friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] - status += "FRIEND_INVITATION_EMAIL_LINK_FOUND " + status += "INVITATION_INFORMATION_FRIEND_INVITATION_EMAIL_LINK_FOUND " + sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id - if friend_invitation_email_link.sender_voter_we_vote_id == voter_we_vote_id: + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED-EMAIL_LINK " error_results = { - 'status': status, - 'success': False, - 'voter_device_id': voter_device_id, - 'friend_first_name': '', - 'friend_last_name': '', - 'friend_image_url_https_large': '', - 'friend_image_url_https_tiny': '', - 'friend_issue_we_vote_id_list': [], - 'friend_we_vote_id': '', - 'friend_organization_we_vote_id': '', - 'invitation_found': invitation_found, - 'invitation_message': '', - 'invitation_secret_key': invitation_secret_key, - 'invitation_secret_key_belongs_to_this_voter': False, + 'status': status, + 'success': False, + 'voter_device_id': voter_device_id, + 'friend_first_name': '', + 'friend_last_name': '', + 'friend_image_url_https_large': '', + 'friend_image_url_https_tiny': '', + 'friend_issue_we_vote_id_list': [], + 'friend_we_vote_id': '', + 'friend_organization_we_vote_id': '', + 'invitation_found': invitation_found, + 'invitation_message': '', + 'invitation_secret_key': invitation_secret_key, + 'invitation_secret_key_belongs_to_this_voter': False, } return error_results if positive_value_exists(sender_voter_we_vote_id): - voter_friend_results = voter_manager.retrieve_voter_by_we_vote_id(sender_voter_we_vote_id) + voter_friend_results = voter_manager.retrieve_voter_by_we_vote_id(sender_voter_we_vote_id, read_only=True) if voter_friend_results['voter_found']: friend_we_vote_id = sender_voter_we_vote_id voter_friend = voter_friend_results['voter'] @@ -983,35 +1090,37 @@ def friend_invitation_information_for_api(voter_device_id, invitation_secret_key status += "MISSING_SENDER_VOTER_WE_VOTE_ID " json_data = { - 'status': status, - 'success': success, - 'voter_device_id': voter_device_id, - 'friend_first_name': friend_first_name, - 'friend_last_name': friend_last_name, - 'friend_image_url_https_large': friend_image_url_https_large, - 'friend_image_url_https_tiny': friend_image_url_https_tiny, - 'friend_issue_we_vote_id_list': friend_issue_we_vote_id_list, - 'friend_we_vote_id': friend_we_vote_id, - 'friend_organization_we_vote_id': friend_organization_we_vote_id, - 'invitation_found': invitation_found, - 'invitation_message': invitation_message, - 'invitation_secret_key': invitation_secret_key, + 'status': status, + 'success': success, + 'voter_device_id': voter_device_id, + 'friend_first_name': friend_first_name, + 'friend_last_name': friend_last_name, + 'friend_image_url_https_large': friend_image_url_https_large, + 'friend_image_url_https_tiny': friend_image_url_https_tiny, + 'friend_issue_we_vote_id_list': friend_issue_we_vote_id_list, + 'friend_we_vote_id': friend_we_vote_id, + 'friend_organization_we_vote_id': friend_organization_we_vote_id, + 'invitation_found': invitation_found, + 'invitation_message': invitation_message, + 'invitation_secret_key': invitation_secret_key, 'invitation_secret_key_belongs_to_this_voter': invitation_secret_key_belongs_to_this_voter, } return json_data -def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify +def friend_acceptance_email_should_be_sent( # friendInvitationByEmailVerify voter_device_id, invitation_secret_key, web_app_root_url=''): """ - + Friendship has been completed and new voter is signed in. Now we want to start the + process of letting the person who invited this friend know that their friend accepted + the invitation. :param voter_device_id: :param invitation_secret_key: :param web_app_root_url: :return: """ status = "" - success = False + success = True # If a voter_device_id is passed in that isn't valid, we want to throw an error device_id_results = is_voter_device_id_valid(voter_device_id) @@ -1020,11 +1129,12 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify json_data = { 'status': status, 'success': False, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': False, - 'invitation_found': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, } return json_data @@ -1033,27 +1143,209 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify error_results = { 'status': status, 'success': True, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': False, - 'invitation_found': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, } return error_results voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) voter_id = voter_results['voter_id'] if not positive_value_exists(voter_id): status += "VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID " error_results = { 'status': status, 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, 'voter_device_id': voter_device_id, 'voter_has_data_to_preserve': False, + } + return error_results + voter = voter_results['voter'] + voter_we_vote_id = voter.we_vote_id + voter_has_data_to_preserve = voter.has_data_to_preserve() + + friend_manager = FriendManager() + friend_invitation_results = friend_manager.retrieve_friend_invitation_from_secret_key( + invitation_secret_key, + for_additional_processes=True, + read_only=False) + if not friend_invitation_results['friend_invitation_found']: + status += "INVITATION_NOT_FOUND_FROM_SECRET_KEY3-SEND_ACCEPTANCE_EMAIL " + error_results = { + 'status': status, + 'success': True, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + # Now that we know we are dealing with a valid friend invitation, extract the info we need + if friend_invitation_results['friend_invitation_voter_link_found']: + invitation_found = True + status += "FRIEND_ACCEPTANCE_FRIEND_INVITATION_VOTER_LINK_FOUND " + friend_invitation_voter_link = friend_invitation_results['friend_invitation_voter_link'] + sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id + voter_we_vote_id_accepting_invitation = friend_invitation_voter_link.recipient_voter_we_vote_id + + if sender_voter_we_vote_id == voter_we_vote_id: + status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " + error_results = { + 'status': status, + 'success': True, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': True, + 'invitation_found': invitation_found, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) + friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + + if not positive_value_exists(friend_invitation_voter_link.invited_friend_accepted_notification_sent): + accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation + original_sender_we_vote_id = sender_voter_we_vote_id + results = friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + web_app_root_url=web_app_root_url) + status += results['status'] + try: + friend_invitation_voter_link.invited_friend_accepted_notification_sent = True + friend_invitation_voter_link.save() + except Exception as e: + status += "COULD_NOT_SAVE_FRIEND_INVITATION_VOTER_LINK: " + str(e) + " " + else: + status += "ALREADY_TRUE: friend_invitation_voter_link.invited_friend_accepted_notification_sent " + elif friend_invitation_results['friend_invitation_email_link_found']: + invitation_found = True + status += "FRIEND_ACCEPTANCE_FRIEND_INVITATION_EMAIL_LINK_FOUND " + friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] + sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id + voter_we_vote_id_accepting_invitation = voter_we_vote_id + + if sender_voter_we_vote_id == voter_we_vote_id: + status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " + error_results = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': True, + 'invitation_found': invitation_found, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return error_results + + friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) + if positive_value_exists(voter_we_vote_id_accepting_invitation): + friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + + if not positive_value_exists(friend_invitation_email_link.invited_friend_accepted_notification_sent): + accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation + original_sender_we_vote_id = sender_voter_we_vote_id + results = friend_accepted_invitation_send( + accepting_voter_we_vote_id, + original_sender_we_vote_id, + web_app_root_url=web_app_root_url) + status = results['status'] + try: + friend_invitation_email_link.invited_friend_accepted_notification_sent = True + friend_invitation_email_link.save() + except Exception as e: + status += "COULD_NOT_SAVE_FRIEND_INVITATION_EMAIL_LINK: " + str(e) + " " + else: + status += "ALREADY_TRUE: friend_invitation_email_link.invited_friend_accepted_notification_sent " + else: + status += "FRIEND_INVITATION_EMAIL_LINK_MISSING-voter_we_vote_id_accepting_invitation " + else: + invitation_found = False + status += "FRIEND_INVITATION_NOT_FOUND " + + json_data = { + 'status': status, + 'success': success, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': invitation_found, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, + } + return json_data + + +def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify + voter_device_id, invitation_secret_key): + """ + + :param voter_device_id: + :param invitation_secret_key: + :return: + """ + status = "" + success = True + + # If a voter_device_id is passed in that isn't valid, we want to throw an error + device_id_results = is_voter_device_id_valid(voter_device_id) + if not device_id_results['success']: + status += "DEVICE_ID_RESULTS_SUCCESS_FALSE " + status += device_id_results['status'] + json_data = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, + } + return json_data + + if not positive_value_exists(invitation_secret_key): + status += "VOTER_EMAIL_ADDRESS_VERIFY_MISSING_SECRET_KEY " + error_results = { + 'status': status, + 'success': True, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, + } + return error_results + + voter_manager = VoterManager() + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=False) + voter_id = voter_results['voter_id'] + if not positive_value_exists(voter_id): + status += "VOTER_NOT_FOUND_FROM_VOTER_DEVICE_ID " + error_results = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, + 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': False, } return error_results voter = voter_results['voter'] @@ -1062,63 +1354,120 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify friend_manager = FriendManager() friend_invitation_results = friend_manager.retrieve_friend_invitation_from_secret_key( - invitation_secret_key, for_accepting_friendship=True, read_only=False) + invitation_secret_key, + for_accepting_friendship=True, + read_only=False) if not friend_invitation_results['friend_invitation_found']: status += "INVITATION_NOT_FOUND_FROM_SECRET_KEY2-ACCEPTING_FRIENDSHIP " error_results = { 'status': status, 'success': True, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': False, + 'invitation_found': False, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return error_results # Now that we have the friend_invitation data, look more closely at it - invitation_found = True - voter_we_vote_id_accepting_invitation = "" + invitation_found = False email_manager = EmailManager() if friend_invitation_results['friend_invitation_voter_link_found']: friend_invitation_voter_link = friend_invitation_results['friend_invitation_voter_link'] + sender_voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id + voter_we_vote_id_accepting_invitation = friend_invitation_voter_link.recipient_voter_we_vote_id + invitation_found = True - if friend_invitation_voter_link.sender_voter_we_vote_id == voter_we_vote_id: + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " error_results = { 'status': status, 'success': True, + 'acceptance_email_should_be_sent': False, + 'attempted_to_approve_own_invitation': True, + 'invitation_found': True, + 'invitation_secret_key': invitation_secret_key, 'voter_device_id': voter_device_id, 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': True, + } + return error_results + + # voter_we_vote_id is probably NOT the same as voter_we_vote_id_accepting_invitation + # We want to make all changes against voter_we_vote_id_accepting_invitation, and those changes will be + # merged into the current voter_we_vote_id on a different API call + recipient_organization_we_vote_id = '' + voter_results = voter_manager.retrieve_voter_by_we_vote_id( + voter_we_vote_id_accepting_invitation, read_only=False) + if not voter_results['voter_found']: + status += "VOTER_THIS_INVITATION_WAS_SENT_TO_COULD_NOT_BE_FOUND: " + voter_results['status'] + " " + error_results = { + 'status': status, + 'success': False, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': True, + 'invitation_found': True, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return error_results - voter_we_vote_id_accepting_invitation = friend_invitation_voter_link.recipient_voter_we_vote_id + voter_accepting_invitation = voter_results['voter'] + recipient_organization_we_vote_id = voter_accepting_invitation.linked_organization_we_vote_id + + # Data healing may be needed + # Even though 'friend_invitation_voter_link_found', we don't necessarily have a previously verified email + # address. By clicking this link, voter_we_vote_id_accepting_invitation is verifying they have access + # to the email used to send this invite. + + # Since the 'friend_invitation_voter_link' doesn't include the email address used, we see if + # voter_accepting_invitation.email has been verified. If not we are going to verify that. + if voter_accepting_invitation.email and not voter_accepting_invitation.email_ownership_is_verified: + # Check to see if the email is owned by anyone else. If not, we are going to assume this is the + # email this voter just verified by clicking the friend invitation + temp_voter_we_vote_id = "" + email_results = email_manager.retrieve_primary_email_with_ownership_verified( + temp_voter_we_vote_id, voter_accepting_invitation.email) + if email_results['email_address_object_found']: + # The email belongs to this or another voter, and we don't want to proceed + # with any additional data healing + pass + else: + # If here, we know that voter_accepting_invitation.email hasn't been claimed by another account + # See if an email_address_object already exists for this voter + this_voter_email_results = email_manager.retrieve_email_address_object( + normalized_email_address=voter_accepting_invitation.email, + voter_we_vote_id=voter_we_vote_id_accepting_invitation) + email_address_object = None + email_address_object_found = False + if this_voter_email_results['email_address_object_found']: + email_address_object = this_voter_email_results['email_address_object'] + email_address_object_found = True + elif this_voter_email_results['email_address_list_found']: + email_address_list = this_voter_email_results['email_address_list'] + email_address_object = email_address_list[0] + email_address_object_found = True + if email_address_object_found: + try: + email_address_object.email_ownership_is_verified = True + email_address_object.save() + + voter_accepting_invitation.email_ownership_is_verified = True + voter_accepting_invitation.primary_email_we_vote_id = email_address_object.we_vote_id + voter_accepting_invitation.save() + except Exception as e: + status += "FAILED_TO_UPDATE_EMAIL_OWNERSHIP_IS_VERIFIED: " + str(e) + " " + # Now we want to make sure we have a current_friend entry - recipient_organization_we_vote_id = '' - voter_results = voter_manager.retrieve_voter_by_we_vote_id( - friend_invitation_voter_link.recipient_voter_we_vote_id) - if voter_results['voter_found']: - recipient_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( - sender_voter_we_vote_id=friend_invitation_voter_link.sender_voter_we_vote_id, - recipient_voter_we_vote_id=friend_invitation_voter_link.recipient_voter_we_vote_id, + sender_voter_we_vote_id=sender_voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, sender_organization_we_vote_id=voter.linked_organization_we_vote_id, recipient_organization_we_vote_id=recipient_organization_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter( - friend_invitation_voter_link.sender_voter_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter( - friend_invitation_voter_link.recipient_voter_we_vote_id) - - accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation - original_sender_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id - results = friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_we_vote_id, - web_app_root_url=web_app_root_url) - status += results['status'] + acceptance_email_should_be_sent = True # Now that a CurrentFriend entry exists, update the FriendInvitation... if friend_results['success']: @@ -1127,7 +1476,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify friend_invitation_voter_link.save() except Exception as e: success = False - status += 'FAILED_TO_UPDATE_INVITATION_STATUS1 ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_INVITATION_STATUS1: ' + str(e) + ' ' else: success = False status += "friend_invitation_voter_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " @@ -1135,16 +1484,20 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # We don't need to do anything with the email because this was an invitation to a known voter elif friend_invitation_results['friend_invitation_email_link_found']: friend_invitation_email_link = friend_invitation_results['friend_invitation_email_link'] - if friend_invitation_email_link.sender_voter_we_vote_id == voter_we_vote_id: + sender_voter_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id + invitation_found = True + + if sender_voter_we_vote_id == voter_we_vote_id: status += "SENDER_AND_RECIPIENT_ARE_IDENTICAL_FAILED " error_results = { 'status': status, 'success': False, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': True, + 'acceptance_email_should_be_sent': False, 'attempted_to_approve_own_invitation': True, + 'invitation_found': True, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return error_results @@ -1167,6 +1520,9 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # We might need to heal the data in the voter record if voter_we_vote_id_accepting_invitation != voter_we_vote_id: + status += "VOTER_ACCEPTING_INVITATION_NOT_CURRENT_SIGNED_IN_VOTER " + email_address_object.email_ownership_is_verified = True + email_address_object.save() email_owner_results = voter_manager.retrieve_voter_by_we_vote_id(email_address_object.voter_we_vote_id) if email_owner_results['voter_found']: email_owner_voter = email_owner_results['voter'] @@ -1174,6 +1530,9 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify else: # If we are here, then the email_address_object doesn't belong to another voter and can be # claimed by this current voter. + status += "VOTER_ACCEPTING_INVITATION_IS_CURRENT_SIGNED_IN_VOTER " + email_address_object.email_ownership_is_verified = True + email_address_object.save() voter_manager.update_voter_email_ownership_verified(voter, email_address_object) if we_have_first_or_last_name_from_friend_invitation_email_link and \ not this_voter_has_first_or_last_name_saved: @@ -1181,6 +1540,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify update_voter_name = True else: + status += "VERIFIED_EMAIL_ADDRESS_OBJECT_NOT_FOUND " voter_we_vote_id_accepting_invitation = voter_we_vote_id # If we are here, we know the email is unclaimed. We can assign it to the current voter. # Is there an email address entry for this voter/email? @@ -1189,6 +1549,7 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify friend_invitation_email_link.recipient_voter_email, email_we_vote_id, voter_we_vote_id) if email_results['email_address_object_found']: + status += "UNVERIFIED_EMAIL_ADDRESS_OBJECT_FOUND " email_address_object = email_results['email_address_object'] try: email_address_object.email_ownership_is_verified = True @@ -1201,8 +1562,9 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify update_voter_name = True except Exception as e: success = False - status += 'FAILED_TO_UPDATE_UNVERIFIED_EMAIL ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_UNVERIFIED_EMAIL: ' + str(e) + ' ' else: + status += "UNVERIFIED_EMAIL_ADDRESS_OBJECT_NOT_FOUND " email_ownership_is_verified = True email_create_results = email_manager.create_email_address_for_voter( friend_invitation_email_link.recipient_voter_email, voter, email_ownership_is_verified) @@ -1229,26 +1591,16 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify # Now that we know who owns the recipient_email_address, update invitation status sender_organization_we_vote_id = '' - voter_results = voter_manager.retrieve_voter_by_we_vote_id( - friend_invitation_email_link.sender_voter_we_vote_id) + voter_results = voter_manager.retrieve_voter_by_we_vote_id(sender_voter_we_vote_id) if voter_results['voter_found']: sender_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( - friend_invitation_email_link.sender_voter_we_vote_id, - voter_we_vote_id_accepting_invitation, - sender_organization_we_vote_id, - voter.linked_organization_we_vote_id - ) - - friend_manager.update_suggested_friends_starting_with_one_voter( - friend_invitation_email_link.sender_voter_we_vote_id) - friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) + sender_voter_we_vote_id=sender_voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, + sender_organization_we_vote_id=sender_organization_we_vote_id, + recipient_organization_we_vote_id=voter.linked_organization_we_vote_id) - accepting_voter_we_vote_id = voter_we_vote_id_accepting_invitation - original_sender_we_vote_id = friend_invitation_email_link.sender_voter_we_vote_id - results = friend_accepted_invitation_send(accepting_voter_we_vote_id, original_sender_we_vote_id, - web_app_root_url=web_app_root_url) - status = results['status'] + acceptance_email_should_be_sent = True if friend_results['success']: try: @@ -1258,12 +1610,12 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify status += ' friend_invitation_email_link_found FRIENDSHIP_CREATED ' except Exception as e: success = False - status += 'FAILED_TO_UPDATE_INVITATION_STATUS2 ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_INVITATION_STATUS2: ' + str(e) + ' ' else: success = False status += "friend_invitation_email_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " - # And finally, create an organization for this brand new signed-in voter so they can create public opinions + # And finally, create an organization for this brand new signed-in voter, so they can create public opinions organization_name = voter.get_full_name() organization_image = voter.voter_photo_url() organization_type = INDIVIDUAL @@ -1284,16 +1636,20 @@ def friend_invitation_by_email_verify_for_api( # friendInvitationByEmailVerify voter.save() status += "VOTER_AND_ORGANIZATION_CREATED_FROM_FRIEND_INVITATION " except Exception as e: - status += "UNABLE_CREATE_AND_LINK_VOTER_FROM_FRIEND_INVITATION " + str(e) + ' ' + status += "UNABLE_CREATE_AND_LINK_VOTER_FROM_FRIEND_INVITATION: " + str(e) + ' ' + else: + acceptance_email_should_be_sent = False + status += "BOTH_FALSE: friend_invitation_email_link_found and friend_invitation_voter_link_found " json_data = { 'status': status, 'success': success, - 'voter_device_id': voter_device_id, - 'voter_has_data_to_preserve': voter_has_data_to_preserve, - 'invitation_found': invitation_found, + 'acceptance_email_should_be_sent': acceptance_email_should_be_sent, 'attempted_to_approve_own_invitation': False, + 'invitation_found': invitation_found, 'invitation_secret_key': invitation_secret_key, + 'voter_device_id': voter_device_id, + 'voter_has_data_to_preserve': voter_has_data_to_preserve, } return json_data @@ -1331,7 +1687,7 @@ def friend_invitation_by_facebook_send_for_api(voter_device_id, recipients_faceb return error_results voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) sender_voter_id = voter_results['voter_id'] if not positive_value_exists(sender_voter_id): error_results = { @@ -1348,7 +1704,7 @@ def friend_invitation_by_facebook_send_for_api(voter_device_id, recipients_faceb friend_manager = FriendManager() facebook_link_to_voter_results = facebook_manager.retrieve_facebook_link_to_voter_from_voter_we_vote_id( - sender_voter_we_vote_id) + sender_voter_we_vote_id, read_only=True) if not facebook_link_to_voter_results['facebook_link_to_voter_found']: status += "FRIEND_INVITATION_BY_FACEBOOK-FACEBOOK_LINK_TO_VOTER_NOT_FOUND " error_results = { @@ -1425,7 +1781,7 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque return error_results voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) voter_id = voter_results['voter_id'] if not positive_value_exists(voter_id): error_results = { @@ -1446,7 +1802,8 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque facebook_manager = FacebookManager() # Retrieve sender voter we vote id - facebook_link_to_voter_results = facebook_manager.retrieve_facebook_link_to_voter(sender_facebook_id) + facebook_link_to_voter_results = facebook_manager.retrieve_facebook_link_to_voter( + sender_facebook_id, read_only=True) if not facebook_link_to_voter_results['facebook_link_to_voter_found']: error_results = { 'status': "FACEBOOK_LINK_TO_SENDER_NOT_FOUND", @@ -1499,11 +1856,10 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque if voter_results['voter_found']: sender_organization_we_vote_id = voter_results['voter'].linked_organization_we_vote_id friend_results = friend_manager.update_or_create_current_friend( - sender_voter_we_vote_id, - voter_we_vote_id_accepting_invitation, - sender_organization_we_vote_id, - voter.linked_organization_we_vote_id, - ) + sender_voter_we_vote_id=sender_voter_we_vote_id, + recipient_voter_we_vote_id=voter_we_vote_id_accepting_invitation, + sender_organization_we_vote_id=sender_organization_we_vote_id, + recipient_organization_we_vote_id=voter.linked_organization_we_vote_id) friend_manager.update_suggested_friends_starting_with_one_voter(sender_voter_we_vote_id) friend_manager.update_suggested_friends_starting_with_one_voter(voter_we_vote_id_accepting_invitation) @@ -1518,7 +1874,7 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque status += "INVITATION_FROM_FACEBOOK_UPDATED " except Exception as e: success = False - status += 'FAILED_TO_UPDATE_INVITATION_STATUS1 ' + str(e) + ' ' + status += 'FAILED_TO_UPDATE_INVITATION_STATUS1: ' + str(e) + ' ' else: success = False status += " friend_invitation_facebook_link_found CREATE_OR_UPDATE_CURRENT_FRIEND_FAILED " @@ -1535,8 +1891,11 @@ def friend_invitation_by_facebook_verify_for_api(voter_device_id, facebook_reque return json_data -def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we_vote_id, invitation_message, - web_app_root_url=''): # friendInvitationByWeVoteIdSend +def friend_invitation_by_we_vote_id_send_for_api( # friendInvitationByWeVoteIdSend + voter_device_id='', + other_voter_we_vote_id='', + invitation_message='', + web_app_root_url=''): """ :param voter_device_id: @@ -1561,7 +1920,7 @@ def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we return error_results voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if not voter_results['voter_found']: error_results = { @@ -1577,7 +1936,7 @@ def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we recipient_voter = Voter() if positive_value_exists(other_voter_we_vote_id): other_voter_we_vote_id_found = True - recipient_voter_results = voter_manager.retrieve_voter_by_we_vote_id(other_voter_we_vote_id) + recipient_voter_results = voter_manager.retrieve_voter_by_we_vote_id(other_voter_we_vote_id, read_only=True) if recipient_voter_results['voter_found']: recipient_voter = recipient_voter_results['voter'] other_voter_found = True @@ -1665,6 +2024,27 @@ def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we email_manager.update_email_address_with_new_subscription_secret_key( email_we_vote_id=recipient_email_we_vote_id) + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/friendinvite" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendinvite/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_FRIEND_REQUESTS_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='friendinvite') + template_variables_for_json = { "subject": subject, "invitation_message": invitation_message, @@ -1674,12 +2054,11 @@ def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we "sender_description": sender_description, "sender_network_details": sender_network_details, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_voter_email, "see_all_friend_requests_url": web_app_root_url_verified + "/friends", - "confirm_friend_request_url": web_app_root_url_verified + "/more/network/key/" + invitation_secret_key, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", + "confirm_friend_request_url": + web_app_root_url_verified + "/more/network/key/" + invitation_secret_key, } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) @@ -1695,7 +2074,10 @@ def friend_invitation_by_we_vote_id_send_for_api(voter_device_id, other_voter_we recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_voter_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " if outbound_results['email_outbound_description_saved']: email_outbound_description = outbound_results['email_outbound_description'] @@ -1979,10 +2361,563 @@ def friend_lists_all_for_api(voter_device_id, # friendListsAll return results -def get_current_friends_list(status, voter): +def generate_mutual_friends_for_all_voters(): + total_mutual_friends_created_count = 0 + total_mutual_friends_updated_count = 0 + total_mutual_friends_update_suppressed_count = 0 + status = "" + success = False + + current_friend_queryset = CurrentFriend.objects.all() + current_friend_queryset = current_friend_queryset.order_by('-date_last_changed') + current_friend_list = list(current_friend_queryset) + # Loop through them CurrentFriend list + for one_current_friend in current_friend_list: + generate_results = generate_mutual_friends_for_current_friend(current_friend=one_current_friend) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + if positive_value_exists(mutual_friends_created_count): + total_mutual_friends_created_count += mutual_friends_created_count + if positive_value_exists(mutual_friends_updated_count): + total_mutual_friends_updated_count += mutual_friends_updated_count + if positive_value_exists(mutual_friends_update_suppressed_count): + total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + + suggested_friend_queryset = SuggestedFriend.objects.all() + suggested_friend_queryset = suggested_friend_queryset.order_by('-date_last_changed') + suggested_friend_list = list(suggested_friend_queryset) + # Loop through them SuggestedFriend list + for one_suggested_friend in suggested_friend_list: + generate_results = generate_mutual_friends_for_suggested_friend(suggested_friend=one_suggested_friend) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + if positive_value_exists(mutual_friends_created_count): + total_mutual_friends_created_count += mutual_friends_created_count + if positive_value_exists(mutual_friends_updated_count): + total_mutual_friends_updated_count += mutual_friends_updated_count + if positive_value_exists(mutual_friends_update_suppressed_count): + total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + + queryset = FriendInvitationVoterLink.objects.all() + queryset = queryset.filter(deleted=False) + queryset = queryset.order_by('-date_last_changed') + friend_invitation_voter_link_list = list(queryset) + # Loop through them SuggestedFriend list + for one_friend_invitation_voter_link in friend_invitation_voter_link_list: + generate_results = generate_mutual_friends_for_friend_invitation_voter_link( + friend_invitation_voter_link=one_friend_invitation_voter_link) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + if positive_value_exists(mutual_friends_created_count): + total_mutual_friends_created_count += mutual_friends_created_count + if positive_value_exists(mutual_friends_updated_count): + total_mutual_friends_updated_count += mutual_friends_updated_count + if positive_value_exists(mutual_friends_update_suppressed_count): + total_mutual_friends_update_suppressed_count += mutual_friends_update_suppressed_count + + if positive_value_exists(total_mutual_friends_created_count): + status += "created: " + str(total_mutual_friends_created_count) + " " + if positive_value_exists(total_mutual_friends_updated_count): + status += "updated: " + str(total_mutual_friends_updated_count) + " " + if positive_value_exists(total_mutual_friends_update_suppressed_count): + status += "update_suppressed: " + str(total_mutual_friends_update_suppressed_count) + " " + + results = { + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_one_voter(voter_we_vote_id='', update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + # ###################### + # Retrieve list of all CurrentFriend entries connected to voter_we_vote_id + current_friend_list_results = friend_manager.retrieve_current_friend_list( + voter_we_vote_id=voter_we_vote_id, + read_only=False) + if not current_friend_list_results['success']: + status += current_friend_list_results['status'] + success = False + results = { + 'success': success, + 'status': status, + } + return results + current_friend_list = current_friend_list_results['current_friend_list'] + for one_current_friend in current_friend_list: + generate_results = generate_mutual_friends_for_current_friend(current_friend=one_current_friend) + status += generate_results['status'] + + # ###################### + # Retrieve list of all SuggestedFriend entries connected to voter_we_vote_id + suggested_friend_list_results = friend_manager.retrieve_suggested_friend_list( + voter_we_vote_id=voter_we_vote_id, + read_only=False) + if not suggested_friend_list_results['success']: + status += suggested_friend_list_results['status'] + success = False + results = { + 'success': success, + 'status': status, + } + return results + suggested_friend_list = suggested_friend_list_results['suggested_friend_list'] + for one_suggested_friend in suggested_friend_list: + generate_results = generate_mutual_friends_for_suggested_friend(suggested_friend=one_suggested_friend) + status += generate_results['status'] + + # ###################### + # Retrieve list of all FriendInvitationVoterLink entries connected to voter_we_vote_id + sent_to_me_results = friend_manager.retrieve_friend_invitations_sent_to_me( + recipient_voter_we_vote_id=voter_we_vote_id, + read_only=False) + if not sent_to_me_results['success']: + status += sent_to_me_results['status'] + success = False + results = { + 'success': success, + 'status': status, + } + return results + friend_invitation_voter_link_list = sent_to_me_results['friend_list'] + for one_friend_invitation_voter_link in friend_invitation_voter_link_list: + generate_results = generate_mutual_friends_for_friend_invitation_voter_link( + friend_invitation_voter_link=one_friend_invitation_voter_link) + status += generate_results['status'] + + results = { + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_current_friend(current_friend=None, update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + voter_we_vote_id = current_friend.viewer_voter_we_vote_id + friend_voter_we_vote_id = current_friend.viewee_voter_we_vote_id + + # Retrieve the voter_we_vote_id's of all mutual friends, based on query of CurrentFriend table + mutual_voter_we_vote_id_list_from_current_friends = \ + friend_manager.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id=voter_we_vote_id, + friend_voter_we_vote_id=friend_voter_we_vote_id) + + # Update CurrentFriend with mutual_friend_count + mutual_friend_count = len(mutual_voter_we_vote_id_list_from_current_friends) + # if positive_value_exists(mutual_friend_count): + # status += "MUTUAL_FRIEND_COUNT: " + str(mutual_friend_count) + " " + + generate_results = generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + mutual_friends_voter_we_vote_id_list_from_current_friends=mutual_voter_we_vote_id_list_from_current_friends, + second_friend_voter_we_vote_id=friend_voter_we_vote_id, + update_existing_data=update_existing_data) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + + change_to_save = False + mutual_friend_count_change = False + if not positive_value_exists(mutual_friend_count) and current_friend.mutual_friend_count is None: + pass + elif current_friend.mutual_friend_count != mutual_friend_count: + current_friend.mutual_friend_count = mutual_friend_count + current_friend.mutual_friend_count_last_updated = localtime(now()).date() # We Vote uses Pacific Time + change_to_save = True + mutual_friend_count_change = True + + always_update_serialized = True + if always_update_serialized or mutual_friend_count_change or update_existing_data: + # Note that in the SuggestedFriend table, we store both "directions" of the suggested friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B + preview_results = generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + second_friend_voter_we_vote_id=friend_voter_we_vote_id) + status += preview_results['status'] + if preview_results['success']: + mutual_friend_preview_list_serialized = preview_results['mutual_friend_preview_list_serialized'] + if not positive_value_exists(mutual_friend_preview_list_serialized) and \ + current_friend.mutual_friend_preview_list_serialized is None: + pass + elif current_friend.mutual_friend_preview_list_serialized != mutual_friend_preview_list_serialized: + current_friend.mutual_friend_preview_list_serialized = mutual_friend_preview_list_serialized + current_friend.mutual_friend_preview_list_update_needed = False + change_to_save = True + else: + status += "FAILED_TO_GENERATE_PREVIEW_LIST_CURRENT_FRIEND " + + if change_to_save: + current_friend.save() + + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_friend_invitation_voter_link( + friend_invitation_voter_link=None, + update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + voter_we_vote_id = friend_invitation_voter_link.sender_voter_we_vote_id + friend_voter_we_vote_id = friend_invitation_voter_link.recipient_voter_we_vote_id + + # Retrieve the voter_we_vote_id's of all mutual friends, based on query of CurrentFriend table + mutual_voter_we_vote_id_list_from_current_friends = \ + friend_manager.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id=voter_we_vote_id, + friend_voter_we_vote_id=friend_voter_we_vote_id) + + # Update SuggestedFriend with mutual_friend_count + mutual_friend_count = len(mutual_voter_we_vote_id_list_from_current_friends) + # if positive_value_exists(mutual_friend_count): + # status += "MUTUAL_FRIEND_COUNT: " + str(mutual_friend_count) + " " + + generate_results = generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + mutual_friends_voter_we_vote_id_list_from_current_friends=mutual_voter_we_vote_id_list_from_current_friends, + second_friend_voter_we_vote_id=friend_voter_we_vote_id, + update_existing_data=update_existing_data) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + + change_to_save = False + mutual_friend_count_change = False + if not positive_value_exists(mutual_friend_count) and friend_invitation_voter_link.mutual_friend_count is None: + pass + elif friend_invitation_voter_link.mutual_friend_count != mutual_friend_count: + friend_invitation_voter_link.mutual_friend_count = mutual_friend_count + friend_invitation_voter_link.mutual_friend_count_last_updated = localtime(now()).date() # Pacific Time + + change_to_save = True + mutual_friend_count_change = True + + always_update_serialized = True + if always_update_serialized or mutual_friend_count_change or update_existing_data: + # Note that in the SuggestedFriend table, we store both "directions" of the suggested friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B + preview_results = generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + second_friend_voter_we_vote_id=friend_voter_we_vote_id) + status += preview_results['status'] + if preview_results['success']: + mutual_friend_preview_list_serialized = preview_results['mutual_friend_preview_list_serialized'] + if not positive_value_exists(mutual_friend_preview_list_serialized) and \ + friend_invitation_voter_link.mutual_friend_preview_list_serialized is None: + pass + elif friend_invitation_voter_link.mutual_friend_preview_list_serialized != \ + mutual_friend_preview_list_serialized: + friend_invitation_voter_link.mutual_friend_preview_list_serialized = \ + mutual_friend_preview_list_serialized + friend_invitation_voter_link.mutual_friend_preview_list_update_needed = False + change_to_save = True + else: + status += "FAILED_TO_GENERATE_PREVIEW_LIST_FRIEND_INVITATION_VOTER_LINK " + + if change_to_save: + friend_invitation_voter_link.save() + + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_suggested_friend(suggested_friend=None, update_existing_data=False): + status = "" + success = True + friend_manager = FriendManager() + + voter_we_vote_id = suggested_friend.viewer_voter_we_vote_id + friend_voter_we_vote_id = suggested_friend.viewee_voter_we_vote_id + + # Retrieve the voter_we_vote_id's of all mutual friends, based on query of CurrentFriend table + mutual_voter_we_vote_id_list_from_current_friends = \ + friend_manager.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id=voter_we_vote_id, + friend_voter_we_vote_id=friend_voter_we_vote_id) + + # Update SuggestedFriend with mutual_friend_count + mutual_friend_count = len(mutual_voter_we_vote_id_list_from_current_friends) + # if positive_value_exists(mutual_friend_count): + # status += "MUTUAL_FRIEND_COUNT: " + str(mutual_friend_count) + " " + + generate_results = generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + mutual_friends_voter_we_vote_id_list_from_current_friends=mutual_voter_we_vote_id_list_from_current_friends, + second_friend_voter_we_vote_id=friend_voter_we_vote_id, + update_existing_data=update_existing_data) + status += generate_results['status'] + mutual_friends_created_count = generate_results['mutual_friends_created_count'] + mutual_friends_updated_count = generate_results['mutual_friends_updated_count'] + mutual_friends_update_suppressed_count = generate_results['mutual_friends_update_suppressed_count'] + + change_to_save = False + mutual_friend_count_change = False + if not positive_value_exists(mutual_friend_count) and suggested_friend.mutual_friend_count is None: + pass + elif suggested_friend.mutual_friend_count != mutual_friend_count: + suggested_friend.mutual_friend_count = mutual_friend_count + suggested_friend.mutual_friend_count_last_updated = localtime(now()).date() # We Vote uses Pacific Time + + change_to_save = True + mutual_friend_count_change = True + + always_update_serialized = True + if always_update_serialized or mutual_friend_count_change or update_existing_data: + # Note that in the SuggestedFriend table, we store both "directions" of the suggested friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B + preview_results = generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id=voter_we_vote_id, + second_friend_voter_we_vote_id=friend_voter_we_vote_id) + status += preview_results['status'] + if preview_results['success']: + mutual_friend_preview_list_serialized = preview_results['mutual_friend_preview_list_serialized'] + if not positive_value_exists(mutual_friend_preview_list_serialized) and \ + suggested_friend.mutual_friend_preview_list_serialized is None: + pass + elif suggested_friend.mutual_friend_preview_list_serialized != mutual_friend_preview_list_serialized: + suggested_friend.mutual_friend_preview_list_serialized = mutual_friend_preview_list_serialized + suggested_friend.mutual_friend_preview_list_update_needed = False + change_to_save = True + else: + status += "FAILED_TO_GENERATE_PREVIEW_LIST_SUGGESTED_FRIEND " + + if change_to_save: + suggested_friend.save() + + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friends_for_two_voters( + first_friend_voter_we_vote_id='', + mutual_friends_voter_we_vote_id_list_from_current_friends=[], + second_friend_voter_we_vote_id='', + update_existing_data=False): friend_manager = FriendManager() + mutual_friends_created_count = 0 + mutual_friends_update_suppressed_count = 0 + mutual_friends_updated_count = 0 + status = "" + success = True voter_manager = VoterManager() - position_metrics_manager = PositionMetricsManager() + + # Retrieve list of MutualFriend entries already existing for this CurrentFriend entry, + # so we can avoid recreating one that already exists. + # AND so we can remove entries which aren't needed any more. + mutual_friend_results = friend_manager.retrieve_mutual_friend_list( + first_friend_voter_we_vote_id=first_friend_voter_we_vote_id, + second_friend_voter_we_vote_id=second_friend_voter_we_vote_id, + read_only=False, # We may need to edit or delete these entries + ) + if not mutual_friend_results['success']: + status += "FAILED_RETRIEVE_MUTUAL_FRIEND_LIST: " + mutual_friend_results['status'] + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + existing_mutual_friend_list = mutual_friend_results['mutual_friend_list'] + + existing_mutual_friend_voter_we_vote_id_list = [] + existing_mutual_friend_dict = {} + # Retrieve the existing MutualFriend entries + for one_mutual_friend in existing_mutual_friend_list: + if positive_value_exists(one_mutual_friend.mutual_friend_voter_we_vote_id): + existing_mutual_friend_voter_we_vote_id_list.append(one_mutual_friend.mutual_friend_voter_we_vote_id) + existing_mutual_friend_dict[one_mutual_friend.mutual_friend_voter_we_vote_id] = one_mutual_friend + + # Loop through all the friends that first_friend_voter_we_vote_id and second_friend_voter_we_vote_id share in common + for one_mutual_friend_voter_we_vote_id in mutual_friends_voter_we_vote_id_list_from_current_friends: + if one_mutual_friend_voter_we_vote_id in existing_mutual_friend_voter_we_vote_id_list: + if positive_value_exists(update_existing_data): + # If we are updating existing data continue + pass + else: + # If not updating existing data, move onto the next mutual friend + mutual_friends_update_suppressed_count += 1 + continue + # Retrieve the voter who is the mutual friend, so we can get the name and profile image + voter_results = voter_manager.retrieve_voter_by_we_vote_id(one_mutual_friend_voter_we_vote_id) + if not voter_results['success'] or not voter_results['voter_found']: + # If we can't retrieve the voter data for this mutual friend for any reason, + # go on to the next mutual friend + status += "FAILED_RETRIEVING_MUTUAL_FRIEND_VOTER: " + voter_results['status'] + " " + continue + mutual_friend_display_name = None + mutual_friend_display_name_exists = False + we_vote_hosted_profile_image_url_medium = None + mutual_friend_profile_image_exists = False + + voter = voter_results['voter'] + if positive_value_exists(voter.get_full_name(real_name_only=True)): + mutual_friend_display_name = voter.get_full_name(real_name_only=True) + mutual_friend_display_name_exists = True + if positive_value_exists(voter.we_vote_hosted_profile_image_url_medium): + we_vote_hosted_profile_image_url_medium = voter.we_vote_hosted_profile_image_url_medium + mutual_friend_profile_image_exists = True + + viewer_to_mutual_friend_friend_count = friend_manager.fetch_mutual_friends_count_from_current_friends( + voter_we_vote_id=first_friend_voter_we_vote_id, + friend_voter_we_vote_id=one_mutual_friend_voter_we_vote_id, + ) + viewee_to_mutual_friend_friend_count = friend_manager.fetch_mutual_friends_count_from_current_friends( + voter_we_vote_id=second_friend_voter_we_vote_id, + friend_voter_we_vote_id=one_mutual_friend_voter_we_vote_id, + ) + + if one_mutual_friend_voter_we_vote_id in existing_mutual_friend_voter_we_vote_id_list: + # Update existing record + if update_existing_data: + mutual_friend = existing_mutual_friend_dict[one_mutual_friend_voter_we_vote_id] + mutual_friends_updated_count += 1 + else: + print("Will not update_existing_data") + else: + # Create new record + mutual_friend = MutualFriend.objects.create( + viewer_voter_we_vote_id=first_friend_voter_we_vote_id, + viewee_voter_we_vote_id=second_friend_voter_we_vote_id, + mutual_friend_voter_we_vote_id=one_mutual_friend_voter_we_vote_id, + mutual_friend_display_name=mutual_friend_display_name, + mutual_friend_display_name_exists=mutual_friend_display_name_exists, + mutual_friend_we_vote_hosted_profile_image_url_medium=we_vote_hosted_profile_image_url_medium, + mutual_friend_profile_image_exists=mutual_friend_profile_image_exists, + viewer_to_mutual_friend_friend_count=viewer_to_mutual_friend_friend_count, + viewee_to_mutual_friend_friend_count=viewee_to_mutual_friend_friend_count, + ) + mutual_friends_created_count += 1 + + # Now delete any MutualFriends in existing_mutual_friend_voter_we_vote_id_list + # which are not in mutual_friends_voter_we_vote_id_list_from_current_friends + mutual_friend_entry_voter_we_vote_id_list_to_delete = \ + list(set(existing_mutual_friend_voter_we_vote_id_list) - + set(mutual_friends_voter_we_vote_id_list_from_current_friends)) + for one_mutual_friend_voter_we_vote_id in mutual_friend_entry_voter_we_vote_id_list_to_delete: + mutual_friend = existing_mutual_friend_dict[one_mutual_friend_voter_we_vote_id] + if hasattr(mutual_friend, 'mutual_friend_voter_we_vote_id'): + mutual_friend.delete() + + # if positive_value_exists(mutual_friends_created_count): + # status += "created: " + str(mutual_friends_created_count) + " " + # if positive_value_exists(mutual_friends_updated_count): + # status += "updated: " + str(mutual_friends_updated_count) + " " + # if positive_value_exists(mutual_friends_update_suppressed_count): + # status += "update_suppressed: " + str(mutual_friends_update_suppressed_count) + " " + results = { + 'mutual_friends_created_count': mutual_friends_created_count, + 'mutual_friends_updated_count': mutual_friends_updated_count, + 'mutual_friends_update_suppressed_count': mutual_friends_update_suppressed_count, + 'success': success, + 'status': status, + } + return results + + +def generate_mutual_friend_preview_list_serialized_for_two_voters( + first_friend_voter_we_vote_id='', + second_friend_voter_we_vote_id=''): + status = "" + success = True + maximum_number_of_mutual_friends_to_return = 8 + + # Note that in the MutualFriend table, we store both "directions" of the mutual friendship: + # person A looking at person B, AND + # person B looking at person A + # We *could* generate a mutual friend preview list unique for each friendship direction (depending on + # whether it is person A viewing, or person B viewing). For now we are creating the same preview list + # for both person A and person B, regardless of which one is looking + queryset = MutualFriend.objects.using('readonly').all() + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=first_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=first_friend_voter_we_vote_id)) + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=second_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=second_friend_voter_we_vote_id)) + queryset = queryset.filter( + Q(mutual_friend_display_name_exists=True) | + Q(mutual_friend_profile_image_exists=True)) + queryset = queryset\ + .annotate(combined_friend_count=F('viewer_to_mutual_friend_friend_count') + + F('viewee_to_mutual_friend_friend_count'))\ + .order_by('-combined_friend_count') + mutual_friend_list = queryset[:maximum_number_of_mutual_friends_to_return] + + mutual_friend_preview_list_found = False + mutual_friend_preview_list = [] + mutual_friend_preview_list_serialized = None + for one_mutual_friend in mutual_friend_list: + one_friend_dict = { + "friend_display_name": one_mutual_friend.mutual_friend_display_name, + "friend_photo_url_medium": one_mutual_friend.mutual_friend_we_vote_hosted_profile_image_url_medium, + } + mutual_friend_preview_list.append(one_friend_dict) + + if len(mutual_friend_preview_list) > 0: + mutual_friend_preview_list_found = True + mutual_friend_preview_list_serialized = json.dumps(mutual_friend_preview_list) + + results = { + 'mutual_friend_preview_list_found': mutual_friend_preview_list_found, + 'mutual_friend_preview_list_serialized': mutual_friend_preview_list_serialized, + 'success': success, + 'status': status, + } + return results + + +def get_current_friends_list(status, voter): + friend_manager = FriendManager() friend_list = [] status += "KIND_OF_LIST-CURRENT_FRIENDS " retrieve_current_friends_as_voters_results = friend_manager.retrieve_current_friends_as_voters( @@ -2017,15 +2952,16 @@ def get_current_friends_list(status, voter): # status += "VOTER_COULD_NOT_BE_HEALED " + heal_results['status'] # else: # status += "COULD_NOT_RETRIEVE_VOTER_THAT_CAN_BE_SAVED " + voter_results['status'] - mutual_friends = friend_manager.fetch_mutual_friends_count(voter.we_vote_id, friend_voter.we_vote_id) + # mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( + # voter.we_vote_id, friend_voter.we_vote_id) # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) one_friend = { "voter_we_vote_id": friend_voter.we_vote_id, "voter_date_last_changed": friend_voter.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), "voter_display_name": friend_voter.get_full_name(), "voter_photo_url_large": friend_voter.we_vote_hosted_profile_image_url_large - if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) - else friend_voter.voter_photo_url(), + if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) + else friend_voter.voter_photo_url(), 'voter_photo_url_medium': friend_voter.we_vote_hosted_profile_image_url_medium, 'voter_photo_url_tiny': friend_voter.we_vote_hosted_profile_image_url_tiny, "voter_email_address": friend_voter.email, @@ -2036,7 +2972,14 @@ def get_current_friends_list(status, voter): "state_code_for_display": friend_voter.state_code_for_display, "invitation_status": "", # Not used with CurrentFriends "invitation_sent_to": "", # Not used with CurrentFriends - "mutual_friends": mutual_friends, + "mutual_friend_count": friend_voter.mutual_friend_count + if hasattr(friend_voter, "mutual_friend_count") and + positive_value_exists(friend_voter.mutual_friend_count) else 0, + "mutual_friend_preview_list": friend_voter.mutual_friend_preview_list + if hasattr(friend_voter, "mutual_friend_preview_list") else [], + "mutual_friends": friend_voter.mutual_friend_count + if hasattr(friend_voter, "mutual_friend_count") and + positive_value_exists(friend_voter.mutual_friend_count) else 0, # "positions_taken": positions_taken, } friend_list.append(one_friend) @@ -2064,7 +3007,7 @@ def get_friend_invitations_processed(status, voter): recipient_voter_email = one_friend_invitation.recipient_voter_email \ if hasattr(one_friend_invitation, "recipient_voter_email") \ else "" - mutual_friends = friend_manager.fetch_mutual_friends_count( + mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( voter.we_vote_id, friend_voter.we_vote_id) # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) @@ -2073,13 +3016,13 @@ def get_friend_invitations_processed(status, voter): "voter_date_last_changed": friend_voter.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), "voter_display_name": friend_voter.get_full_name(), "voter_photo_url_large": friend_voter.we_vote_hosted_profile_image_url_large - if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) - else friend_voter.voter_photo_url(), + if positive_value_exists(friend_voter.we_vote_hosted_profile_image_url_large) + else friend_voter.voter_photo_url(), 'voter_photo_url_medium': friend_voter.we_vote_hosted_profile_image_url_medium, 'voter_photo_url_tiny': friend_voter.we_vote_hosted_profile_image_url_tiny, "voter_email_address": friend_voter.email, "voter_twitter_handle": friend_voter.twitter_screen_name, - "voter_twitter_description": "", # To be implemented + "voter_twitter_description": "", # To be implemented "voter_twitter_followers_count": 0, # To be implemented "linked_organization_we_vote_id": friend_voter.linked_organization_we_vote_id, "state_code_for_display": friend_voter.state_code_for_display, @@ -2104,6 +3047,7 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): status += retrieve_invitations_sent_to_me_results['status'] if retrieve_invitations_sent_to_me_results['friend_list_found']: raw_friend_list = retrieve_invitations_sent_to_me_results['friend_list'] + friend_invitation_dict = {} heal_results = heal_friend_invitations_sent_to_me(voter.we_vote_id, raw_friend_list) invitation_table_dict = {} verified_friend_list = heal_results['friend_list'] @@ -2111,6 +3055,10 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): sent_to_me_voter_we_vote_id_list = [] recipient_voter_email_dict = {} for one_friend_invitation in verified_friend_list: + friend_invitation_dict[one_friend_invitation.sender_voter_we_vote_id] = \ + one_friend_invitation \ + if hasattr(one_friend_invitation, "mutual_friend_count") \ + else "" sent_to_me_voter_we_vote_id_list.append(one_friend_invitation.sender_voter_we_vote_id) invitation_table_dict[one_friend_invitation.sender_voter_we_vote_id] = \ one_friend_invitation.invitation_table \ @@ -2127,8 +3075,28 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): sent_to_me_friend_list = results['voter_list'] # Augment the line with voter information for friend_voter in sent_to_me_friend_list: # This is the voter who sent the invitation to me - mutual_friends = friend_manager.fetch_mutual_friends_count( + mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( voter.we_vote_id, friend_voter.we_vote_id) + if friend_voter.we_vote_id in friend_invitation_dict: + mutual_friend_count = friend_invitation_dict[friend_voter.we_vote_id].mutual_friend_count \ + if hasattr(friend_invitation_dict[friend_voter.we_vote_id], "mutual_friend_count") and \ + positive_value_exists(friend_invitation_dict[friend_voter.we_vote_id].mutual_friend_count) \ + else 0 + mutual_friend_preview_list_serialized = \ + friend_invitation_dict[friend_voter.we_vote_id].mutual_friend_preview_list_serialized \ + if hasattr(friend_invitation_dict[friend_voter.we_vote_id], + "mutual_friend_preview_list_serialized") \ + and positive_value_exists(friend_invitation_dict[friend_voter.we_vote_id] + .mutual_friend_preview_list_serialized) \ + else None + if mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + else: + mutual_friend_count = 0 + mutual_friend_preview_list = [] # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) one_friend = { @@ -2149,6 +3117,8 @@ def get_friend_invitations_sent_to_me(status, voter, read_only=True): "invitation_status": "", # Not used for invitations sent to me "invitation_sent_to": recipient_voter_email_dict[friend_voter.we_vote_id], "invitation_table": invitation_table_dict[friend_voter.we_vote_id], + "mutual_friend_count": mutual_friend_count, + "mutual_friend_preview_list": mutual_friend_preview_list, "mutual_friends": mutual_friends, # "positions_taken": positions_taken, } @@ -2201,10 +3171,12 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): status += retrieve_invitations_sent_by_me_results['status'] if retrieve_invitations_sent_by_me_results['friend_list_found']: raw_friend_list = retrieve_invitations_sent_by_me_results['friend_list'] + friend_invitation_dict = {} invitation_status_dict = {} invitation_table_dict = {} recipient_voter_email_dict = {} recipient_voter_email_invitations = [] + recipient_voter_we_vote_id = '' recipient_voter_we_vote_id_list = [] for one_friend_invitation in raw_friend_list: # Two kinds of invitations come in the raw_friend_list, 1) an invitation connected to voter @@ -2214,6 +3186,10 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): else: recipient_voter_we_vote_id = "" if positive_value_exists(recipient_voter_we_vote_id): + friend_invitation_dict[recipient_voter_we_vote_id] = \ + one_friend_invitation \ + if hasattr(one_friend_invitation, "mutual_friend_count") \ + else "" recipient_voter_we_vote_id_list.append(recipient_voter_we_vote_id) invitation_status_dict[recipient_voter_we_vote_id] = \ one_friend_invitation.invitation_status \ @@ -2238,8 +3214,29 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): for friend_voter in sent_by_me_friend_list: # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(friend_voter) - mutual_friends = friend_manager.fetch_mutual_friends_count( - voter.we_vote_id, friend_voter.we_vote_id) + mutual_friends = friend_manager.fetch_mutual_friends_count_from_current_friends( + voter.we_vote_id, recipient_voter_we_vote_id) + if recipient_voter_we_vote_id in friend_invitation_dict: + mutual_friend_count = friend_invitation_dict[recipient_voter_we_vote_id].mutual_friend_count \ + if hasattr(friend_invitation_dict[recipient_voter_we_vote_id], "mutual_friend_count") and \ + positive_value_exists(friend_invitation_dict[recipient_voter_we_vote_id] + .mutual_friend_count) \ + else 0 + mutual_friend_preview_list_serialized = \ + friend_invitation_dict[recipient_voter_we_vote_id].mutual_friend_preview_list_serialized \ + if hasattr(friend_invitation_dict[recipient_voter_we_vote_id], + "mutual_friend_preview_list_serialized") \ + and positive_value_exists(friend_invitation_dict[recipient_voter_we_vote_id] + .mutual_friend_preview_list_serialized) \ + else None + if mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + else: + mutual_friend_count = 0 + mutual_friend_preview_list = [] one_friend = { "voter_we_vote_id": friend_voter.we_vote_id, "voter_date_last_changed": friend_voter.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), @@ -2258,6 +3255,8 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): "invitation_status": invitation_status_dict[friend_voter.we_vote_id], "invitation_table": invitation_table_dict[friend_voter.we_vote_id], "invitation_sent_to": recipient_voter_email_dict[friend_voter.we_vote_id], + "mutual_friend_count": mutual_friend_count, + "mutual_friend_preview_list": mutual_friend_preview_list, "mutual_friends": mutual_friends, # "positions_taken": positions_taken, } @@ -2275,7 +3274,7 @@ def get_friend_invitations_sent_by_me(status, voter, read_only=True): "voter_twitter_handle": "", "voter_twitter_description": "", # To be implemented "voter_twitter_followers_count": 0, # To be implemented - "state_code_for_display": friend_voter.state_code_for_display, + "state_code_for_display": "", # To be implemented "voter_email_address": one_friend_invitation.recipient_voter_email, "invitation_status": one_friend_invitation.invitation_status, "invitation_table": one_friend_invitation.invitation_table, @@ -2317,8 +3316,9 @@ def get_suggested_friends_list(status, voter): status += "SUGGESTED_FRIEND_VOTER_COULD_NOT_BE_HEALED " + heal_results['status'] else: status += "SUGGESTED-COULD_NOT_RETRIEVE_VOTER_THAT_CAN_BE_SAVED " + voter_results['status'] - mutual_friends = \ - friend_manager.fetch_mutual_friends_count(voter.we_vote_id, suggested_friend.we_vote_id) + # mutual_friends = \ + # friend_manager.fetch_mutual_friends_count_from_current_friends( + # voter.we_vote_id, suggested_friend.we_vote_id) # Removed for now for speed # positions_taken = position_metrics_manager.fetch_positions_count_for_this_voter(suggested_friend) one_friend = { @@ -2326,8 +3326,8 @@ def get_suggested_friends_list(status, voter): "voter_date_last_changed": suggested_friend.date_last_changed.strftime('%Y-%m-%d %H:%M:%S'), "voter_display_name": suggested_friend.get_full_name(), "voter_photo_url_large": suggested_friend.we_vote_hosted_profile_image_url_large - if positive_value_exists(suggested_friend.we_vote_hosted_profile_image_url_large) - else suggested_friend.voter_photo_url(), + if positive_value_exists(suggested_friend.we_vote_hosted_profile_image_url_large) + else suggested_friend.voter_photo_url(), 'voter_photo_url_medium': suggested_friend.we_vote_hosted_profile_image_url_medium, 'voter_photo_url_tiny': suggested_friend.we_vote_hosted_profile_image_url_tiny, "voter_email_address": suggested_friend.email, @@ -2338,7 +3338,14 @@ def get_suggested_friends_list(status, voter): "state_code_for_display": suggested_friend.state_code_for_display, "invitation_status": "", # Not used with SuggestedFriendList "invitation_sent_to": "", # Not used with SuggestedFriendList - "mutual_friends": mutual_friends, + "mutual_friend_count": suggested_friend.mutual_friend_count + if hasattr(suggested_friend, "mutual_friend_count") and + positive_value_exists(suggested_friend.mutual_friend_count) else 0, + "mutual_friend_preview_list": suggested_friend.mutual_friend_preview_list + if hasattr(suggested_friend, "mutual_friend_preview_list") else [], + "mutual_friends": suggested_friend.mutual_friend_count + if hasattr(suggested_friend, "mutual_friend_count") and + positive_value_exists(suggested_friend.mutual_friend_count) else 0, # "positions_taken": positions_taken, } friend_list.append(one_friend) @@ -2548,6 +3555,27 @@ def message_to_friend_send_for_api( email_manager.update_email_address_with_new_subscription_secret_key( email_we_vote_id=recipient_email_we_vote_id) + # Unsubscribe link in email + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # recipient_email_subscription_secret_key, + recipient_unsubscribe_url = \ + "{root_url}/unsubscribe/{email_secret_key}/friendmessage" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=web_app_root_url_verified, + ) + # Instant unsubscribe link in email header + list_unsubscribe_url = \ + "{root_url}/apis/v1/unsubscribeInstant/{email_secret_key}/friendmessage/" \ + "".format( + email_secret_key=recipient_email_subscription_secret_key, + root_url=WE_VOTE_SERVER_ROOT_URL, + ) + # Instant unsubscribe email address in email header + # from voter.models import NOTIFICATION_FRIEND_MESSAGES_EMAIL + list_unsubscribe_mailto = "unsubscribe@wevote.us?subject=unsubscribe%20{setting}" \ + "".format(setting='friendmessage') + template_variables_for_json = { "subject": subject, "message_to_friend": message_to_friend, @@ -2557,11 +3585,9 @@ def message_to_friend_send_for_api( "sender_description": sender_description, "sender_network_details": sender_network_details, "recipient_name": recipient_name, + "recipient_unsubscribe_url": recipient_unsubscribe_url, "recipient_voter_email": recipient_voter_email, "see_ballot_url": web_app_root_url_verified + "/ballot", - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - recipient_email_subscription_secret_key, - "email_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/emailOpen?email_key=1234", } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) @@ -2575,7 +3601,10 @@ def message_to_friend_send_for_api( recipient_email_we_vote_id=recipient_email_we_vote_id, recipient_voter_email=recipient_voter_email, template_variables_in_json=template_variables_in_json, - kind_of_email_template=kind_of_email_template) + kind_of_email_template=kind_of_email_template, + list_unsubscribe_mailto=list_unsubscribe_mailto, + list_unsubscribe_url=list_unsubscribe_url, + ) status += outbound_results['status'] + " " if outbound_results['email_outbound_description_saved']: email_outbound_description = outbound_results['email_outbound_description'] @@ -2594,7 +3623,9 @@ def message_to_friend_send_for_api( # 'friend_invite_sent': True, # } # suggested_results = friend_manager.update_suggested_friend( - # voter_we_vote_id=sender_voter.we_vote_id, other_voter_we_vote_id=other_voter_we_vote_id, defaults=defaults) + # voter_we_vote_id=sender_voter.we_vote_id, + # other_voter_we_vote_id=other_voter_we_vote_id, + # defaults=defaults) # status += suggested_results['status'] results = { @@ -2649,7 +3680,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ friend_invitation_email_link_from_sender_list = \ friend_invitation_email_link_from_sender_results['friend_invitation_list'] - # TO SENDER: Now get existing invitations for the to_voter so we can see if an invitation already exists + # TO SENDER: Now get existing invitations for the to_voter, so we can see if an invitation already exists friend_invitation_email_link_to_sender_results = friend_manager.retrieve_friend_invitation_email_link_list( to_voter_we_vote_id) friend_invitation_email_link_to_sender_list = \ @@ -2671,7 +3702,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ from_sender_entry.save() friend_invitation_entries_moved += 1 except Exception as e: - status += "FriendInvitationEmailLink Sender entries not moved " + str(e) + ' ' + status += "FriendInvitationEmailLink Sender entries not moved: " + str(e) + ' ' friend_invitation_entries_not_moved += 1 else: status += "to_sender_invitation_found found, EmailLink Sender entries not moved " @@ -2688,7 +3719,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ friend_invitation_voter_link_from_sender_list = \ friend_invitation_voter_link_from_sender_results['friend_invitation_list'] - # TO SENDER: Now get existing invitations for the to_voter so we can see if an invitation already exists + # TO SENDER: Now get existing invitations for the to_voter, so we can see if an invitation already exists friend_invitation_voter_link_to_sender_results = friend_manager.retrieve_friend_invitation_voter_link_list( to_voter_we_vote_id) friend_invitation_voter_link_to_sender_list = \ @@ -2710,7 +3741,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ from_sender_entry.save() friend_invitation_entries_moved += 1 except Exception as e: - status += "FriendInvitationVoterLink Sender entries not moved " + str(e) + ' ' + status += "FriendInvitationVoterLink Sender entries not moved: " + str(e) + ' ' friend_invitation_entries_not_moved += 1 else: status += "to_sender_invitation_found found, VoterLink Sender entries not moved " @@ -2725,7 +3756,7 @@ def move_friend_invitations_to_another_voter(from_voter_we_vote_id, to_voter_we_ friend_invitation_voter_link_from_recipient_list = \ friend_invitation_voter_link_from_recipient_results['friend_invitation_list'] - # TO RECIPIENT: Now get existing invitations for the to_voter so we can see if an invitation already exists + # TO RECIPIENT: Now get existing invitations for the to_voter, so we can see if an invitation already exists friend_invitation_voter_link_to_recipient_results = friend_manager.retrieve_friend_invitation_voter_link_list( '', to_voter_we_vote_id) friend_invitation_voter_link_to_recipient_list = \ @@ -2806,9 +3837,9 @@ def move_friends_to_another_voter( voter_manager.fetch_linked_organization_we_vote_id_by_voter_we_vote_id(to_voter_we_vote_id) friend_manager = FriendManager() - from_friend_results = friend_manager.retrieve_current_friends(from_voter_we_vote_id, read_only=False) + from_friend_results = friend_manager.retrieve_current_friend_list(from_voter_we_vote_id, read_only=False) from_friend_list = from_friend_results['current_friend_list'] - to_friend_results = friend_manager.retrieve_current_friends(to_voter_we_vote_id, read_only=False) + to_friend_results = friend_manager.retrieve_current_friend_list(to_voter_we_vote_id, read_only=False) to_friend_list = to_friend_results['current_friend_list'] for from_friend_entry in from_friend_list: @@ -2835,17 +3866,18 @@ def move_friends_to_another_voter( friend_entries_moved += 1 except Exception as e: friend_entries_not_moved += 1 - status += "PROBLEM_UPDATING_FRIEND " + str(e) + ' ' + status += "PROBLEM_UPDATING_FRIEND: " + str(e) + ' ' - from_friend_list_remaining_results = friend_manager.retrieve_current_friends(from_voter_we_vote_id, read_only=False) + from_friend_list_remaining_results = friend_manager.retrieve_current_friend_list( + from_voter_we_vote_id, + read_only=False) from_friend_list_remaining = from_friend_list_remaining_results['current_friend_list'] for from_friend_entry in from_friend_list_remaining: # Delete the remaining friendship values try: - # Leave this turned off until testing is finished from_friend_entry.delete() except Exception as e: - status += "PROBLEM_DELETING_FRIEND " + str(e) + ' ' + status += "PROBLEM_DELETING_FRIEND: " + str(e) + ' ' results = { 'status': status, @@ -2919,7 +3951,7 @@ def move_suggested_friends_to_another_voter( suggested_friend_entries_moved += 1 except Exception as e: suggested_friend_entries_not_moved += 1 - status += "PROBLEM_UPDATING_SUGGESTED_FRIEND " + str(e) + ' ' + status += "PROBLEM_UPDATING_SUGGESTED_FRIEND: " + str(e) + ' ' from_friend_list_remaining_results = friend_manager.retrieve_suggested_friend_list( from_voter_we_vote_id, hide_deleted=False, read_only=False) @@ -2930,7 +3962,7 @@ def move_suggested_friends_to_another_voter( # Leave this turned off until testing is finished from_friend_entry.delete() except Exception as e: - status += "PROBLEM_DELETING_FRIEND " + str(e) + ' ' + status += "PROBLEM_DELETING_FRIEND: " + str(e) + ' ' results = { 'status': status, diff --git a/friend/models.py b/friend/models.py index a715579c7..28a1312eb 100644 --- a/friend/models.py +++ b/friend/models.py @@ -60,6 +60,13 @@ class CurrentFriend(models.Model): verbose_name="voter we vote id person 2", max_length=255, null=True, blank=True, unique=False, db_index=True) viewee_organization_we_vote_id = models.CharField( max_length=255, null=True, blank=True, unique=False, db_index=True) + + mutual_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + mutual_friend_count_last_updated = models.DateTimeField(null=True) + + mutual_friend_preview_list_serialized = models.TextField(default=None, null=True) + mutual_friend_preview_list_update_needed = models.BooleanField(default=True) + date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) def fetch_other_organization_we_vote_id(self, one_we_vote_id): @@ -99,6 +106,7 @@ class FriendInvitationEmailLink(models.Model): recipient_last_name = models.CharField(verbose_name='last name', max_length=255, null=True, blank=True) secret_key = models.CharField( verbose_name="secret key to accept invite", max_length=255, null=True, blank=True, unique=True) + invited_friend_accepted_notification_sent = models.BooleanField(default=False) invitation_message = models.TextField(null=True, blank=True) invitation_status = models.CharField(max_length=50, choices=INVITATION_STATUS_CHOICES, default=NO_RESPONSE) date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) @@ -156,11 +164,19 @@ class FriendInvitationVoterLink(models.Model): max_length=255, null=True, blank=True, unique=False, db_index=True) secret_key = models.CharField( verbose_name="secret key to accept invite", max_length=255, null=True, blank=True, unique=True) + invited_friend_accepted_notification_sent = models.BooleanField(default=False) invitation_message = models.TextField(null=True, blank=True) invitation_status = models.CharField(max_length=50, choices=INVITATION_STATUS_CHOICES, default=NO_RESPONSE) date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True, db_index=True) merge_by_secret_key_allowed = models.BooleanField(default=True) # To allow merges after delete invitation_table = models.CharField(max_length=8, default=FRIEND_INVITATION_VOTER_LINK) + + mutual_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + mutual_friend_count_last_updated = models.DateTimeField(null=True) + + mutual_friend_preview_list_serialized = models.TextField(default=None, null=True) + mutual_friend_preview_list_update_needed = models.BooleanField(default=True) + deleted = models.BooleanField(default=False) # If invitation is completed or rescinded, mark as deleted @@ -767,17 +783,20 @@ def fetch_current_friends_count(self, voter_we_vote_id): current_friends_count = 0 return current_friends_count - def fetch_mutual_friends_count(self, voter_we_vote_id, friend_we_vote_id): + def fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + self, + voter_we_vote_id='', + friend_voter_we_vote_id=''): """ TODO: This could be converted to database only calculation for better speed :param voter_we_vote_id: - :param friend_we_vote_id: + :param friend_voter_we_vote_id: :return: """ - mutual_friends_count = 0 + mutual_friends_voter_we_vote_id_list = [] - if not positive_value_exists(voter_we_vote_id) or not positive_value_exists(friend_we_vote_id): - return mutual_friends_count + if not positive_value_exists(voter_we_vote_id) or not positive_value_exists(friend_voter_we_vote_id): + return mutual_friends_voter_we_vote_id_list voter_friends_we_vote_id_list = [] friend_friends_we_vote_id_list = [] @@ -797,20 +816,29 @@ def fetch_mutual_friends_count(self, voter_we_vote_id, friend_we_vote_id): try: friend_friends_queryset = CurrentFriend.objects.using('readonly').all() friend_friends_queryset = friend_friends_queryset.filter( - Q(viewer_voter_we_vote_id__iexact=friend_we_vote_id) | - Q(viewee_voter_we_vote_id__iexact=friend_we_vote_id)) + Q(viewer_voter_we_vote_id__iexact=friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=friend_voter_we_vote_id)) friend_friends_list = list(friend_friends_queryset) for one_friend in friend_friends_list: - friend_friends_we_vote_id_list.append(one_friend.fetch_other_voter_we_vote_id(friend_we_vote_id)) + friend_friends_we_vote_id_list.append(one_friend.fetch_other_voter_we_vote_id(friend_voter_we_vote_id)) except Exception as e: - mutual_friends_count = 0 - return mutual_friends_count + mutual_friends_voter_we_vote_id_list = [] + return mutual_friends_voter_we_vote_id_list voter_set = set(voter_friends_we_vote_id_list) friend_set = set(friend_friends_we_vote_id_list) mutual_set = voter_set & friend_set - if mutual_set: - mutual_friends_count = len(mutual_set) + mutual_friends_voter_we_vote_id_list = list(mutual_set) + return mutual_friends_voter_we_vote_id_list + + def fetch_mutual_friends_count_from_current_friends(self, voter_we_vote_id='', friend_voter_we_vote_id=''): + mutual_friends_count = 0 + mutual_friends_voter_we_vote_id_list = \ + self.fetch_mutual_friends_voter_we_vote_id_list_from_current_friends( + voter_we_vote_id, friend_voter_we_vote_id) + + if mutual_friends_voter_we_vote_id_list: + mutual_friends_count = len(mutual_friends_voter_we_vote_id_list) return mutual_friends_count @@ -830,14 +858,14 @@ def fetch_suggested_friends_count(self, voter_we_vote_id): suggested_friends_count = 0 return suggested_friends_count - def retrieve_current_friends(self, voter_we_vote_id, read_only=True): + def retrieve_current_friend_list(self, voter_we_vote_id, read_only=True): status = "" current_friend_list = [] # The entries from CurrentFriend table current_friend_list_found = False if not positive_value_exists(voter_we_vote_id): success = False - status += 'VALID_VOTER_WE_VOTE_ID_MISSING' + status += 'VALID_VOTER_WE_VOTE_ID_MISSING ' results = { 'success': success, 'status': status, @@ -856,7 +884,7 @@ def retrieve_current_friends(self, voter_we_vote_id, read_only=True): Q(viewer_voter_we_vote_id__iexact=voter_we_vote_id) | Q(viewee_voter_we_vote_id__iexact=voter_we_vote_id)) current_friend_queryset = current_friend_queryset.order_by('-date_last_changed') - current_friend_list = current_friend_queryset + current_friend_list = list(current_friend_queryset) if len(current_friend_list): success = True @@ -875,7 +903,7 @@ def retrieve_current_friends(self, voter_we_vote_id, read_only=True): except Exception as e: success = False current_friend_list_found = False - status += 'FAILED retrieve_current_friends ' + str(e) + " " + status += 'FAILED retrieve_current_friend_list: ' + str(e) + " " current_friend_list = [] results = { @@ -942,17 +970,32 @@ def retrieve_current_friends_as_voters(self, voter_we_vote_id, read_only=True): status += 'AS_VOTERS_FAILED retrieve_current_friends_as_voters ' + str(e) + " " if current_friend_list_found: - current_friend_we_vote_id_list = [] + current_friend_dict = {} + current_friend_voter_we_vote_id_list = [] for current_friend_entry in current_friend_list: we_vote_id_of_friend = current_friend_entry.fetch_other_voter_we_vote_id(voter_we_vote_id) - current_friend_we_vote_id_list.append(we_vote_id_of_friend) + current_friend_voter_we_vote_id_list.append(we_vote_id_of_friend) + current_friend_dict[we_vote_id_of_friend] = current_friend_entry voter_manager = VoterManager() results = voter_manager.retrieve_voter_list_by_we_vote_id_list( - voter_we_vote_id_list=current_friend_we_vote_id_list, + voter_we_vote_id_list=current_friend_voter_we_vote_id_list, read_only=read_only) if results['voter_list_found']: - friend_list = results['voter_list'] + friend_list = [] + raw_friend_list = results['voter_list'] friend_list_found = True + # Augment friend_list with mutual_friend data + for one_voter in raw_friend_list: + if one_voter.we_vote_id in current_friend_dict: + current_friend = current_friend_dict[one_voter.we_vote_id] + one_voter.mutual_friend_count = current_friend.mutual_friend_count + if current_friend.mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(current_friend.mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + one_voter.mutual_friend_preview_list = mutual_friend_preview_list + friend_list.append(one_voter) results = { 'success': success, @@ -965,7 +1008,7 @@ def retrieve_current_friends_as_voters(self, voter_we_vote_id, read_only=True): def retrieve_friends_we_vote_id_list(self, voter_we_vote_id): """ - This is similar to retrieve_current_friends, but only returns the we_vote_id + This is similar to retrieve_current_friend_list, but only returns the we_vote_id :param voter_we_vote_id: :return: """ @@ -1856,12 +1899,18 @@ def retrieve_friend_invitations_sent_to_me(self, recipient_voter_we_vote_id, rea return results def retrieve_friend_invitation_from_secret_key( - self, invitation_secret_key, for_accepting_friendship=False, for_merge_accounts=False, - for_retrieving_information=False, read_only=True): + self, + invitation_secret_key, + for_accepting_friendship=False, + for_additional_processes=False, + for_merge_accounts=False, + for_retrieving_information=False, + read_only=True): """ :param invitation_secret_key: :param for_accepting_friendship: + :param for_additional_processes: :param for_merge_accounts: :param for_retrieving_information: :param read_only: @@ -1886,6 +1935,9 @@ def retrieve_friend_invitation_from_secret_key( if positive_value_exists(for_accepting_friendship): voter_link_query = voter_link_query.exclude(invitation_status__iexact=ACCEPTED) status += "FOR_ACCEPTING_FRIENDSHIP " + elif positive_value_exists(for_additional_processes): + voter_link_query = voter_link_query.filter(invitation_status__iexact=ACCEPTED) + status += "FOR_ADDITIONAL_PROCESSES " elif positive_value_exists(for_merge_accounts): voter_link_query = voter_link_query.filter(merge_by_secret_key_allowed=True) status += "FOR_MERGE_ACCOUNTS " @@ -1933,6 +1985,9 @@ def retrieve_friend_invitation_from_secret_key( if positive_value_exists(for_accepting_friendship): email_link_query = email_link_query.exclude(invitation_status__iexact=ACCEPTED) status += "FOR_ACCEPTING_FRIENDSHIP " + elif positive_value_exists(for_additional_processes): + email_link_query = email_link_query.filter(invitation_status__iexact=ACCEPTED) + status += "FOR_ADDITIONAL_PROCESSES " elif positive_value_exists(for_merge_accounts): email_link_query = email_link_query.filter(merge_by_secret_key_allowed=True) status += "FOR_MERGE_ACCOUNTS " @@ -2006,6 +2061,73 @@ def retrieve_friend_invitation_from_facebook(self, facebook_request_id, recipien } return results + def retrieve_mutual_friend_list( + self, + first_friend_voter_we_vote_id='', + second_friend_voter_we_vote_id='', + mutual_friend_voter_we_vote_id='', + read_only=True): + status = "" + success = True + mutual_friend_list = [] # The entries from MutualFriend table + mutual_friend_list_found = False + + if positive_value_exists(first_friend_voter_we_vote_id) and \ + positive_value_exists(second_friend_voter_we_vote_id): + pass + elif positive_value_exists(mutual_friend_voter_we_vote_id): + pass + else: + success = False + status += 'RETRIEVE_MUTUAL_FRIEND_LIST_MISSING_KEY_VARIABLE ' + results = { + 'success': success, + 'status': status, + 'mutual_friend_list_found': mutual_friend_list_found, + 'mutual_friend_list': mutual_friend_list, + } + return results + + try: + if positive_value_exists(read_only): + queryset = MutualFriend.objects.using('readonly').all() + else: + queryset = MutualFriend.objects.all() + if positive_value_exists(first_friend_voter_we_vote_id): + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=first_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=first_friend_voter_we_vote_id)) + if positive_value_exists(second_friend_voter_we_vote_id): + queryset = queryset.filter( + Q(viewer_voter_we_vote_id__iexact=second_friend_voter_we_vote_id) | + Q(viewee_voter_we_vote_id__iexact=second_friend_voter_we_vote_id)) + if positive_value_exists(mutual_friend_voter_we_vote_id): + queryset = queryset.filter( + mutual_friend_voter_we_vote_id=mutual_friend_voter_we_vote_id, + ) + queryset = queryset.order_by('-date_last_changed') + mutual_friend_list = list(queryset) + + if len(mutual_friend_list): + mutual_friend_list_found = True + status += 'MUTUAL_FRIEND_LIST_RETRIEVED ' + else: + mutual_friend_list_found = False + status += 'NO_MUTUAL_FRIEND_LIST_RETRIEVED ' + except Exception as e: + success = False + mutual_friend_list_found = False + status += 'FAILED retrieve_mutual_friend_list: ' + str(e) + " " + mutual_friend_list = [] + + results = { + 'success': success, + 'status': status, + 'mutual_friend_list_found': mutual_friend_list_found, + 'mutual_friend_list': mutual_friend_list, + } + return results + def unfriend_current_friend(self, acting_voter_we_vote_id, other_voter_we_vote_id): # Retrieve the existing friendship status = "" @@ -2189,9 +2311,11 @@ def retrieve_suggested_friend_list_as_voters(self, voter_we_vote_id='', read_onl filtered_suggested_friend_list_we_vote_ids = {} if suggested_friend_list_found: + suggested_friend_dict = {} voter_manager = VoterManager() for suggested_friend_entry in suggested_friend_list: we_vote_id_of_friend = suggested_friend_entry.fetch_other_voter_we_vote_id(voter_we_vote_id) + suggested_friend_dict[we_vote_id_of_friend] = suggested_friend_entry if we_vote_id_of_friend in friends_we_vote_id_list: # If this person is already a friend, don't suggest as a friend @@ -2217,8 +2341,21 @@ def retrieve_suggested_friend_list_as_voters(self, voter_we_vote_id='', read_onl voter_we_vote_id_list=ordered_suggested_friend_list_we_vote_ids, read_only=read_only) if results['voter_list_found']: - friend_list = results['voter_list'] + friend_list = [] + raw_friend_list = results['voter_list'] friend_list_found = True + # Augment friend_list with mutual_friend data + for one_voter in raw_friend_list: + if one_voter.we_vote_id in suggested_friend_dict: + suggested_friend = suggested_friend_dict[one_voter.we_vote_id] + one_voter.mutual_friend_count = suggested_friend.mutual_friend_count + if suggested_friend.mutual_friend_preview_list_serialized: + mutual_friend_preview_list = \ + json.loads(suggested_friend.mutual_friend_preview_list_serialized) + else: + mutual_friend_preview_list = [] + one_voter.mutual_friend_preview_list = mutual_friend_preview_list + friend_list.append(one_voter) results = { 'success': success, @@ -2232,7 +2369,7 @@ def retrieve_suggested_friend_list_as_voters(self, voter_we_vote_id='', read_onl def update_suggested_friends_starting_with_one_voter(self, starting_voter_we_vote_id, read_only=False): """ Note that we default to "read_only=False" (that is, the live db) since usually we are doing this update - right after adding a friend, and we want the new friend to be returned in "retrieve_current_friends". + right after adding a friend, and we want the new friend to be returned in "retrieve_current_friend_list". We use the live database ("read_only=False") because we don't want to create a race condition with the replicated read_only not being caught up with the master fast enough (since a friend was just created above.) @@ -2241,7 +2378,7 @@ def update_suggested_friends_starting_with_one_voter(self, starting_voter_we_vot :param read_only: :return: """ - all_friends_one_person_results = self.retrieve_current_friends(starting_voter_we_vote_id, read_only=read_only) + all_friends_one_person_results = self.retrieve_current_friend_list(starting_voter_we_vote_id, read_only=read_only) suggested_friend_created_count = 0 if all_friends_one_person_results['current_friend_list_found']: current_friend_list = all_friends_one_person_results['current_friend_list'] @@ -2336,6 +2473,42 @@ def update_suggested_friend(self, voter_we_vote_id, other_voter_we_vote_id, return results +class MutualFriend(models.Model): + """ + This is considered a "cache" table, with all data being generated and aggregated to speed up other processes. + This table helps us generate and update the array stored in CurrentFriend.mutual_friend_preview_list_serialized. + The "direction" doesn't matter, although it usually indicates who initiated the first friend invitation. + """ + viewer_voter_we_vote_id = models.CharField( + verbose_name="voter we vote id person 1", max_length=255, null=True, blank=True, unique=False, db_index=True) + viewee_voter_we_vote_id = models.CharField( + verbose_name="voter we vote id person 2", max_length=255, null=True, blank=True, unique=False, db_index=True) + mutual_friend_voter_we_vote_id = models.CharField( + max_length=255, null=True, blank=True, unique=False, db_index=True) + + mutual_friend_display_name = models.CharField(max_length=255, null=True, blank=True) + mutual_friend_display_name_exists = models.BooleanField(default=False, db_index=True) + + mutual_friend_we_vote_hosted_profile_image_url_medium = models.TextField(blank=True, null=True) + mutual_friend_profile_image_exists = models.BooleanField(default=False, db_index=True) + + # The more friends the viewer and the mutual_friend share, the more important this mutual friend is + viewer_to_mutual_friend_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + # The more friends the viewee and the mutual_friend share, the more important this mutual friend is + viewee_to_mutual_friend_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + + date_last_changed = models.DateTimeField(null=True, auto_now=True) + + def fetch_other_voter_we_vote_id(self, one_we_vote_id): + if one_we_vote_id == self.viewer_voter_we_vote_id: + return self.viewee_voter_we_vote_id + elif one_we_vote_id == self.viewee_voter_we_vote_id: + return self.viewer_voter_we_vote_id + else: + # If the we_vote_id passed in wasn't found, don't return another we_vote_id + return "" + + class SuggestedFriend(models.Model): """ This table stores possible friend connections. @@ -2352,6 +2525,13 @@ class SuggestedFriend(models.Model): verbose_name="second voter to remove suggested friend", max_length=255, null=True, blank=True, unique=False) friend_invite_sent = models.BooleanField(default=False) current_friends = models.BooleanField(default=False) + + mutual_friend_count = models.PositiveSmallIntegerField(null=True, unique=False) + mutual_friend_count_last_updated = models.DateTimeField(null=True) + + mutual_friend_preview_list_serialized = models.TextField(default=None, null=True) + mutual_friend_preview_list_update_needed = models.BooleanField(default=True) + date_last_changed = models.DateTimeField(verbose_name='date last changed', null=True, auto_now=True) def fetch_other_voter_we_vote_id(self, one_we_vote_id): diff --git a/friend/urls.py b/friend/urls.py index 597bb3137..d2c807ec1 100644 --- a/friend/urls.py +++ b/friend/urls.py @@ -8,9 +8,12 @@ urlpatterns = [ - url(r'^$', views_admin.batches_home_view, name='batches_home',), - # url(r'^batch_action_list/$', views_admin.batch_action_list_view, name='batch_action_list'), - # url(r'^batch_action_list_process/$', views_admin.batch_action_list_process_view, name='batch_action_list_process'), - # url(r'^batch_list/$', views_admin.batch_list_view, name='batch_list'), - # url(r'^batch_list_process/$', views_admin.batch_list_process_view, name='batch_list_process'), + url(r'^current_friends_data_healing/$', + views_admin.current_friends_data_healing_view, name='current_friends_data_healing'), + url(r'^generate_mutual_friends_for_all_voters/$', + views_admin.generate_mutual_friends_for_all_voters_view, name='generate_mutual_friends_for_all_voters'), + url(r'^generate_mutual_friends_for_one_voter/$', + views_admin.generate_mutual_friends_for_one_voter_view, name='generate_mutual_friends_for_one_voter'), + url(r'^refresh_voter_friend_count/$', + views_admin.refresh_voter_friend_count_view, name='refresh_voter_friend_count'), ] diff --git a/friend/views_admin.py b/friend/views_admin.py index 366b3c15e..d9e27c076 100644 --- a/friend/views_admin.py +++ b/friend/views_admin.py @@ -2,7 +2,151 @@ # Brought to you by We Vote. Be good. # -*- coding: UTF-8 -*- +from django.contrib import messages +from django.contrib.auth.decorators import login_required +from django.http import HttpResponseRedirect +from django.urls import reverse + +from .controllers import generate_mutual_friends_for_all_voters, generate_mutual_friends_for_one_voter +from .models import CurrentFriend, FriendManager import wevote_functions.admin +from admin_tools.views import redirect_to_sign_in_page +from wevote_functions.functions import convert_to_int, positive_value_exists +from voter.models import Voter, voter_has_authority, VoterManager, voter_setup logger = wevote_functions.admin.get_logger(__name__) + +@login_required +def current_friends_data_healing_view(request): + status = "" + number_of_linked_org_updates = 0 + number_of_voters_missing_linked_org = 0 + voter_we_vote_id_list = [] + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + + try: + query = Voter.objects.all() + query = query.filter(friend_count__gt=0) + query = query.values_list('we_vote_id', flat=True).distinct() + voter_we_vote_id_list = list(query) + except Exception as e: + messages.add_message(request, messages.ERROR, + 'Could not retrieve any voters with current_friend > 0: ' + str(e)) + + if len(voter_we_vote_id_list): + voter_manager = VoterManager() + for voter_we_vote_id in voter_we_vote_id_list: + linked_organization_we_vote_id = voter_manager.fetch_linked_organization_we_vote_id_by_voter_we_vote_id( + voter_we_vote_id=voter_we_vote_id) + if positive_value_exists(linked_organization_we_vote_id): + viewer_updated = CurrentFriend.objects \ + .filter(viewer_voter_we_vote_id__iexact=voter_we_vote_id) \ + .exclude(viewer_organization_we_vote_id__iexact=linked_organization_we_vote_id) \ + .update(viewer_organization_we_vote_id=linked_organization_we_vote_id) + if positive_value_exists(viewer_updated): + number_of_linked_org_updates += viewer_updated + + viewee_updated = CurrentFriend.objects \ + .filter(viewee_voter_we_vote_id__iexact=voter_we_vote_id) \ + .exclude(viewee_organization_we_vote_id__iexact=linked_organization_we_vote_id) \ + .update(viewee_organization_we_vote_id=linked_organization_we_vote_id) + if positive_value_exists(viewee_updated): + number_of_linked_org_updates += viewee_updated + else: + number_of_voters_missing_linked_org += 1 + + if number_of_linked_org_updates: + messages.add_message(request, messages.INFO, + 'linked_organization_we_vote_id updates: ' + str(number_of_linked_org_updates)) + else: + messages.add_message(request, messages.INFO, + 'No linked_organization_we_vote_id updates. ') + + if number_of_voters_missing_linked_org: + messages.add_message(request, messages.ERROR, + 'number_of_voters_missing_linked_org: ' + str(number_of_voters_missing_linked_org)) + + return HttpResponseRedirect(reverse('voter:voter_list', args=())) + + +@login_required +def generate_mutual_friends_for_all_voters_view(request): + status = "" + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + + results = generate_mutual_friends_for_all_voters() + status += results['status'] + messages.add_message(request, messages.INFO, 'status: ' + str(status)) + + return HttpResponseRedirect(reverse('voter:voter_list', args=())) + + +@login_required +def generate_mutual_friends_for_one_voter_view(request): + status = "" + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + + voter_id = request.GET.get('voter_id', 0) + voter_id = convert_to_int(voter_id) + voter_we_vote_id = request.GET.get('voter_we_vote_id', '') + + results = generate_mutual_friends_for_one_voter(voter_we_vote_id=voter_we_vote_id) + status += results['status'] + messages.add_message(request, messages.INFO, 'status: ' + str(status)) + + return HttpResponseRedirect(reverse('voter:voter_edit', args=(voter_id,))) + + +@login_required +def refresh_voter_friend_count_view(request): + """ + + :param request: + :return: + """ + status = "" + + # admin, analytics_admin, partner_organization, political_data_manager, political_data_viewer, verified_volunteer + authority_required = {'admin'} # We may want to add a "voter_admin" + if not voter_has_authority(request, authority_required): + return redirect_to_sign_in_page(request, authority_required) + + query = CurrentFriend.objects.using('readonly').all() + query = query.values_list('viewer_voter_we_vote_id', flat=True).distinct() + viewer_voter_we_vote_id_list = list(query) + + query = CurrentFriend.objects.using('readonly').all() + query = query.values_list('viewee_voter_we_vote_id', flat=True).distinct() + viewee_voter_we_vote_id_list = list(query) + + voter_we_vote_id_list = list(set(viewer_voter_we_vote_id_list + viewee_voter_we_vote_id_list)) + + friend_manager = FriendManager() + voter_manager = VoterManager() + voters_with_friends = 0 + for one_voter_we_vote_id in voter_we_vote_id_list: + friends_count = friend_manager.fetch_current_friends_count(voter_we_vote_id=one_voter_we_vote_id) + if positive_value_exists(friends_count): + voters_with_friends += 1 + results = voter_manager.retrieve_voter_by_we_vote_id(voter_we_vote_id=one_voter_we_vote_id) + if results['voter_found']: + results['voter'].friend_count = friends_count + results['voter'].save() + + if voters_with_friends: + messages.add_message(request, messages.INFO, 'voters_with_friends: ' + str(voters_with_friends)) + + return HttpResponseRedirect(reverse('voter:voter_list', args=())) diff --git a/image/controllers.py b/image/controllers.py index 9b73b8f8b..fbd1833be 100644 --- a/image/controllers.py +++ b/image/controllers.py @@ -529,7 +529,7 @@ def cache_voter_master_images(voter_id): # else: facebook_manager = FacebookManager() facebook_link_to_voter_results = facebook_manager.retrieve_facebook_link_to_voter_from_voter_we_vote_id( - voter.we_vote_id) + voter.we_vote_id, read_only=True) if facebook_link_to_voter_results['facebook_link_to_voter_found']: facebook_id = facebook_link_to_voter_results['facebook_link_to_voter'].facebook_user_id diff --git a/import_export_ballotpedia/controllers.py b/import_export_ballotpedia/controllers.py index 9b78952ec..b47763d22 100644 --- a/import_export_ballotpedia/controllers.py +++ b/import_export_ballotpedia/controllers.py @@ -2746,7 +2746,9 @@ def process_ballotpedia_voter_districts(google_civic_election_id, state_code, mo # Look for any measures in this election with this ballotpedia_district_id results = measure_list_manager.retrieve_measures( - google_civic_election_id, one_district['ballotpedia_district_id'], state_code=state_code) + google_civic_election_id=google_civic_election_id, + ballotpedia_district_id=one_district['ballotpedia_district_id'], + state_code=state_code) if results['measure_list_found']: measure_list_objects = results['measure_list_objects'] for one_measure in measure_list_objects: diff --git a/import_export_ballotpedia/views_admin.py b/import_export_ballotpedia/views_admin.py index 0698d35fd..1b803f0c5 100644 --- a/import_export_ballotpedia/views_admin.py +++ b/import_export_ballotpedia/views_admin.py @@ -11,11 +11,9 @@ from datetime import date from django.contrib import messages from django.contrib.auth.decorators import login_required -from django.contrib.messages import get_messages from django.urls import reverse from django.db.models import Q from django.http import HttpResponseRedirect -from django.shortcuts import render from election.models import Election, ElectionManager from import_export_batches.models import BatchSet, BATCH_SET_SOURCE_IMPORT_BALLOTPEDIA_BALLOT_ITEMS @@ -36,8 +34,6 @@ MEASURE = 'MEASURE' POLITICIAN = 'POLITICIAN' -MAP_POINTS_RETRIEVED_EACH_BATCH_CHUNK = 125 # 125. Formerly 250 and 111 - @login_required def attach_ballotpedia_election_view(request, election_local_id=0): diff --git a/import_export_batches/controllers.py b/import_export_batches/controllers.py index 4be215871..160d964b0 100755 --- a/import_export_batches/controllers.py +++ b/import_export_batches/controllers.py @@ -63,7 +63,7 @@ def create_batch_row_actions( office_objects_dict={}): """ Cycle through all BatchRow entries for this batch_header_id and move the values we can find into - the BatchRowActionYYY table so we can review it before importing it + the BatchRowActionYYY table, so we can review it before importing it :param batch_header_id: :param batch_description: :param batch_row_id: @@ -254,6 +254,9 @@ def create_batch_row_actions( # batch_row_action_politician = results['batch_row_action_politician'] number_of_batch_actions_created += 1 success = True + status += "CREATE_BATCH_ROW_ACTION_POSITION-START: " + status += results['status'] + print_to_log(logger=logger, exception_message_optional=status) elif kind_of_batch == IMPORT_BALLOT_ITEM: results = create_batch_row_action_ballot_item( batch_description=batch_description, @@ -1343,7 +1346,8 @@ def create_batch_row_action_contest_office(batch_description, batch_header_map, matching_results = candidate_list_manager.retrieve_candidates_from_non_unique_identifiers( google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, - candidate_name=candidate_name) + candidate_name=candidate_name, + read_only=True) if matching_results['candidate_found']: candidate = matching_results['candidate'] @@ -2193,7 +2197,8 @@ def create_batch_row_action_candidate(batch_description, batch_header_map, one_b google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, candidate_twitter_handle=candidate_twitter_handle, - candidate_name=candidate_name) + candidate_name=candidate_name, + read_only=True) if matching_results['candidate_found']: candidate = matching_results['candidate'] @@ -2284,7 +2289,8 @@ def create_batch_row_action_candidate(batch_description, batch_header_map, one_b contest_office_name=contest_office_name, google_civic_election_id=google_civic_election_id, incoming_state_code=state_code, - district_id=office_district_id) + district_id=office_district_id, + read_only=True) if matching_results['contest_office_found']: contest_office = matching_results['contest_office'] contest_office_name = contest_office.office_name @@ -2569,8 +2575,7 @@ def create_batch_row_action_position(batch_description, batch_header_map, one_ba contest_office_manager = ContestOfficeManager() if positive_value_exists(candidate_we_vote_id): candidate_manager = CandidateManager() - candidate_results = candidate_manager.retrieve_candidate_from_we_vote_id( - candidate_we_vote_id) + candidate_results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if candidate_results['candidate_found']: candidate = candidate_results['candidate'] @@ -2609,7 +2614,8 @@ def create_batch_row_action_position(batch_description, batch_header_map, one_ba google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, candidate_twitter_handle=candidate_twitter_handle, - candidate_name=candidate_name) + candidate_name=candidate_name, + read_only=True) if matching_results['candidate_found']: candidate = matching_results['candidate'] @@ -2755,6 +2761,7 @@ def create_batch_row_action_position(batch_description, batch_header_map, one_ba kind_of_action = IMPORT_CREATE else: kind_of_action = IMPORT_TO_BE_DETERMINED + print_to_log(logger=logger, exception_message_optional=status) try: batch_row_action_position.batch_set_id = batch_description.batch_set_id @@ -2945,13 +2952,13 @@ def create_batch_row_action_ballot_item(batch_description, positive_value_exists(candidate_twitter_handle) or positive_value_exists(candidate_name): candidate_list_manager = CandidateListManager() google_civic_election_id_list = [google_civic_election_id] - # Needs to be read_only=False so we don't get "terminating connection due to conflict with recovery" error + # Needs to be read_only=False, so we don't get "terminating connection due to conflict with recovery" error matching_results = candidate_list_manager.retrieve_candidates_from_non_unique_identifiers( google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, candidate_twitter_handle=candidate_twitter_handle, candidate_name=candidate_name, - read_only=False) + read_only=True) if matching_results['candidate_found']: candidate = matching_results['candidate'] keep_looking_for_duplicates = False @@ -5239,7 +5246,7 @@ def import_ballot_item_data_from_batch_row_actions(batch_header_id, batch_row_id return results try: - batch_description = BatchDescription.objects.get(batch_header_id=batch_header_id) + batch_description = BatchDescription.objects.using('readonly').get(batch_header_id=batch_header_id) batch_description_found = True except BatchDescription.DoesNotExist: batch_description = BatchDescription() @@ -5262,7 +5269,7 @@ def import_ballot_item_data_from_batch_row_actions(batch_header_id, batch_row_id # kind_of_batch = batch_description.kind_of_batch try: - batch_header_map = BatchHeaderMap.objects.get(batch_header_id=batch_header_id) + batch_header_map = BatchHeaderMap.objects.using('readonly').get(batch_header_id=batch_header_id) batch_header_map_found = True except BatchHeaderMap.DoesNotExist: batch_header_map_found = False @@ -5507,7 +5514,7 @@ def delete_ballot_item_data_from_batch_row_actions(batch_header_id, ballot_item_ return results try: - batch_description = BatchDescription.objects.get(batch_header_id=batch_header_id) + batch_description = BatchDescription.objects.using('readonly').get(batch_header_id=batch_header_id) batch_description_found = True except BatchDescription.DoesNotExist: # This is fine @@ -5525,7 +5532,7 @@ def delete_ballot_item_data_from_batch_row_actions(batch_header_id, ballot_item_ return results try: - batch_header_map = BatchHeaderMap.objects.get(batch_header_id=batch_header_id) + batch_header_map = BatchHeaderMap.objects.using('readonly').get(batch_header_id=batch_header_id) batch_header_map_found = True except BatchHeaderMap.DoesNotExist: # This is fine diff --git a/import_export_batches/controllers_batch_process.py b/import_export_batches/controllers_batch_process.py index 145324813..41de714b7 100755 --- a/import_export_batches/controllers_batch_process.py +++ b/import_export_batches/controllers_batch_process.py @@ -11,7 +11,7 @@ CALCULATE_SITEWIDE_DAILY_METRICS, \ CALCULATE_SITEWIDE_ELECTION_METRICS, \ CALCULATE_SITEWIDE_VOTER_METRICS, \ - IMPORT_CREATE, IMPORT_DELETE, \ + GENERATE_VOTER_GUIDES, IMPORT_CREATE, IMPORT_DELETE, \ RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, \ REFRESH_BALLOT_ITEMS_FROM_VOTERS, SEARCH_TWITTER_FOR_CANDIDATE_TWITTER_HANDLE, UPDATE_TWITTER_DATA_FROM_TWITTER from activity.controllers import process_activity_notice_seeds_triggered_by_batch_process @@ -22,8 +22,10 @@ from analytics.models import AnalyticsManager from api_internal_cache.models import ApiInternalCacheManager from ballot.models import BallotReturnedListManager -from datetime import timedelta -from django.utils.timezone import now +from candidate.models import CandidateListManager +from datetime import datetime, timedelta +from django.db.models import Q +from django.utils.timezone import localtime, now from election.models import ElectionManager from exception.models import handle_exception from import_export_twitter.controllers import fetch_number_of_candidates_needing_twitter_search, \ @@ -32,13 +34,15 @@ retrieve_possible_twitter_handles_in_bulk from issue.controllers import update_issue_statistics import json +from position.models import PositionEntered from voter_guide.controllers import voter_guides_upcoming_retrieve_for_api +from voter_guide.models import VoterGuideManager, VoterGuidesGenerated import wevote_functions.admin -from wevote_functions.functions import positive_value_exists +from wevote_functions.functions import convert_to_int, positive_value_exists from wevote_settings.models import fetch_batch_process_system_on, fetch_batch_process_system_activity_notices_on, \ fetch_batch_process_system_api_refresh_on, fetch_batch_process_system_ballot_items_on, \ fetch_batch_process_system_calculate_analytics_on, fetch_batch_process_system_search_twitter_on, \ - fetch_batch_process_system_update_twitter_on + fetch_batch_process_system_generate_voter_guides_on, fetch_batch_process_system_update_twitter_on logger = wevote_functions.admin.get_logger(__name__) @@ -222,8 +226,9 @@ def process_next_ballot_items(): REFRESH_BALLOT_ITEMS_FROM_VOTERS, RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS] - # Retrieve list of all ballot item BatchProcesses which have been started but not completed so we can decide + # Retrieve list of all ballot item BatchProcesses which have been started but not completed, so we can decide # our next steps + # TODO: This logic needs to be looked at. How do we know this batch isn't already running on another process? results = batch_process_manager.retrieve_batch_process_list( kind_of_process_list=ballot_item_kind_of_processes, process_active=True, @@ -417,6 +422,9 @@ def process_next_general_maintenance(): CALCULATE_ORGANIZATION_DAILY_METRICS, CALCULATE_ORGANIZATION_ELECTION_METRICS] kind_of_processes_to_run = kind_of_processes_to_run + analytics_process_list + if fetch_batch_process_system_generate_voter_guides_on(): + generate_voter_guides_process_list = [GENERATE_VOTER_GUIDES] + kind_of_processes_to_run = kind_of_processes_to_run + generate_voter_guides_process_list if fetch_batch_process_system_search_twitter_on(): search_twitter_process_list = [SEARCH_TWITTER_FOR_CANDIDATE_TWITTER_HANDLE] kind_of_processes_to_run = kind_of_processes_to_run + search_twitter_process_list @@ -532,6 +540,77 @@ def process_next_general_maintenance(): status=status, ) + # ############################ + # Generate voter guides - make sure we have a voter guide for every Organization + Election pair + # if an endorsement has been added since the last voter guides were generated + if not fetch_batch_process_system_generate_voter_guides_on(): + status += "BATCH_PROCESS_SYSTEM_GENERATE_VOTER_GUIDES_TURNED_OFF " + else: + # We only want one GENERATE_VOTER_GUIDES process to be running at a time + generate_voter_guides_process_is_already_in_queue = False + for batch_process in batch_process_list_already_scheduled: + if batch_process.kind_of_process in [GENERATE_VOTER_GUIDES]: + status += "GENERATE_VOTER_GUIDES_ALREADY_SCHEDULED(" + str(batch_process.id) + ") " + generate_voter_guides_process_is_already_in_queue = True + for batch_process in batch_process_list_already_running: + if batch_process.kind_of_process in [GENERATE_VOTER_GUIDES]: + status += "GENERATE_VOTER_GUIDES_ALREADY_RUNNING(" + str(batch_process.id) + ") " + generate_voter_guides_process_is_already_in_queue = True + if generate_voter_guides_process_is_already_in_queue: + pass + else: + # Get list of upcoming elections we should generate voter guides for (minus those already generated) + election_ids_that_need_voter_guides_generated = [] + election_manager = ElectionManager() + results = election_manager.retrieve_upcoming_google_civic_election_id_list() + if results['upcoming_google_civic_election_id_list_found']: + upcoming_google_civic_election_id_list = results['upcoming_google_civic_election_id_list'] + # Make sure they are all integer + upcoming_google_civic_election_id_list_converted = [] + for one_google_civic_election_id in upcoming_google_civic_election_id_list: + upcoming_google_civic_election_id_list_converted\ + .append(convert_to_int(one_google_civic_election_id)) + upcoming_google_civic_election_id_list = upcoming_google_civic_election_id_list_converted + if positive_value_exists(len(upcoming_google_civic_election_id_list)): + # Get list of elections we have generated voter guides for in the last 24 hours (so we can remove) + time_threshold = localtime(now() - timedelta(hours=24)).date() # Pacific Time for TIME_ZONE + query = VoterGuidesGenerated.objects.using('readonly').all() + query = query.filter(date_last_changed__gte=time_threshold) + election_ids_already_generated_list = query.values_list('google_civic_election_id', + flat=True).distinct() + + if positive_value_exists(len(election_ids_already_generated_list)): + election_ids_that_need_voter_guides_generated = \ + list(set(upcoming_google_civic_election_id_list) - set(election_ids_already_generated_list)) + else: + election_ids_that_need_voter_guides_generated = upcoming_google_civic_election_id_list + + if positive_value_exists(len(election_ids_that_need_voter_guides_generated)): + first_election_id = election_ids_that_need_voter_guides_generated[0] + status += "CREATING_GENERATE_VOTER_GUIDES_BATCH_PROCESS_FOR_ELECTION-" + str(first_election_id) + " " + results = batch_process_manager.create_batch_process( + google_civic_election_id=first_election_id, + kind_of_process=GENERATE_VOTER_GUIDES) + status += results['status'] + success = results['success'] + if results['batch_process_saved']: + batch_process = results['batch_process'] + status += "SCHEDULED_NEW_GENERATE_VOTER_GUIDES " + batch_process_manager.create_batch_process_log_entry( + batch_process_id=batch_process.id, + kind_of_process=batch_process.kind_of_process, + google_civic_election_id=batch_process.google_civic_election_id, + status=status, + ) + else: + status += "FAILED_TO_SCHEDULE-" + str(GENERATE_VOTER_GUIDES) + " " + batch_process_manager.create_batch_process_log_entry( + batch_process_id=0, + kind_of_process=GENERATE_VOTER_GUIDES, + google_civic_election_id=first_election_id, + status=status, + ) + # ############################ # Twitter Search - Possible Twitter Handle Matches if not fetch_batch_process_system_search_twitter_on(): @@ -762,6 +841,9 @@ def process_next_general_maintenance(): elif batch_process.kind_of_process in [CALCULATE_SITEWIDE_DAILY_METRICS]: results = process_one_sitewide_daily_analytics_batch_process(batch_process) status += results['status'] + elif batch_process.kind_of_process in [GENERATE_VOTER_GUIDES]: + results = process_one_generate_voter_guides_batch_process(batch_process) + status += results['status'] elif batch_process.kind_of_process in [SEARCH_TWITTER_FOR_CANDIDATE_TWITTER_HANDLE]: results = process_one_search_twitter_batch_process(batch_process, status=status) status = results['status'] # Not additive since we pass status into function @@ -2111,6 +2193,128 @@ def process_activity_notice_batch_process(batch_process): return results +def process_one_generate_voter_guides_batch_process(batch_process): + status = "" + success = True + voter_guide_manager = VoterGuideManager() + batch_process_manager = BatchProcessManager() + + kind_of_process = batch_process.kind_of_process + + # When a batch_process is running, we mark when it was "taken off the shelf" to be worked on. + # When the process is complete, we should reset this to "NULL" + try: + batch_process.date_started = now() + batch_process.date_checked_out = now() + batch_process.save() + except Exception as e: + status += "ERROR-GENERATE_VOTER_GUIDES-CHECKED_OUT_TIME_NOT_SAVED: " + str(e) + " " + handle_exception(e, logger=logger, exception_message=status) + success = False + batch_process_manager.create_batch_process_log_entry( + batch_process_id=batch_process.id, + kind_of_process=kind_of_process, + google_civic_election_id=batch_process.google_civic_election_id, + status=status, + ) + results = { + 'success': success, + 'status': status, + } + return results + + # Generate voter guides for one election + google_civic_election_id = batch_process.google_civic_election_id + + # Query PositionEntered table in this election for unique organization_we_vote_ids + candidate_list_manager = CandidateListManager() + results = candidate_list_manager.retrieve_candidate_we_vote_id_list_from_election_list( + google_civic_election_id_list=[google_civic_election_id]) + if not positive_value_exists(results['success']): + success = False + candidate_we_vote_id_list = results['candidate_we_vote_id_list'] + + positions_exist_query = PositionEntered.objects.using('readonly').all() + positions_exist_query = positions_exist_query.filter( + Q(google_civic_election_id=google_civic_election_id) | + Q(candidate_campaign_we_vote_id__in=candidate_we_vote_id_list)) + positions_exist_query = positions_exist_query.filter( + Q(vote_smart_rating__isnull=True) | Q(vote_smart_rating="")) + # NOTE: There is a bug here to address -- this is not returning a list of distinct 'organization_we_vote_id' values + positions_exist_query = positions_exist_query.values_list('organization_we_vote_id', flat=True).distinct() + organization_we_vote_ids_with_positions = list(positions_exist_query) + # status += str(organization_we_vote_ids_with_positions) + # Add extra filter for safety while figuring out why distinct didn't work + organization_we_vote_ids_with_positions_filtered = [] + for organization_we_vote_id in organization_we_vote_ids_with_positions: + if organization_we_vote_id not in organization_we_vote_ids_with_positions_filtered: + organization_we_vote_ids_with_positions_filtered.append(organization_we_vote_id) + + elections_dict = {} + voter_guides_generated_count = 0 + for organization_we_vote_id in organization_we_vote_ids_with_positions_filtered: + results = voter_guide_manager.update_or_create_organization_voter_guide_by_election_id( + organization_we_vote_id=organization_we_vote_id, + google_civic_election_id=google_civic_election_id, + elections_dict=elections_dict, + ) + if results['success']: + voter_guides_generated_count += 1 + else: + status += results['status'] + elections_dict = results['elections_dict'] + + if success: + status += "VOTER_GUIDES_GENERATED_COUNT: " + str(voter_guides_generated_count) + " " + results = voter_guide_manager.update_or_create_voter_guides_generated( + google_civic_election_id=google_civic_election_id, + number_of_voter_guides=voter_guides_generated_count, + ) + status += results['status'] + + try: + batch_process.completion_summary = status + batch_process.date_checked_out = None + batch_process.date_completed = now() + batch_process.save() + + batch_process_manager.create_batch_process_log_entry( + batch_process_id=batch_process.id, + kind_of_process=kind_of_process, + google_civic_election_id=google_civic_election_id, + status=status, + ) + except Exception as e: + status += "ERROR-VOTER_GUIDES_GENERATED_DATE_COMPLETED_TIME_NOT_SAVED: " + str(e) + " " + handle_exception(e, logger=logger, exception_message=status) + batch_process_manager.create_batch_process_log_entry( + batch_process_id=batch_process.id, + kind_of_process=kind_of_process, + google_civic_election_id=google_civic_election_id, + status=status, + ) + results = { + 'success': success, + 'status': status, + } + return results + else: + status += "VOTER_GUIDES_GENERATED_FAILED " + success = False + batch_process_manager.create_batch_process_log_entry( + batch_process_id=batch_process.id, + kind_of_process=kind_of_process, + google_civic_election_id=google_civic_election_id, + status=status, + ) + + results = { + 'success': success, + 'status': status, + } + return results + + def process_one_search_twitter_batch_process(batch_process, status=""): success = True batch_process_manager = BatchProcessManager() diff --git a/import_export_batches/models.py b/import_export_batches/models.py index 819afcd3b..0532a8b9f 100644 --- a/import_export_batches/models.py +++ b/import_export_batches/models.py @@ -604,6 +604,7 @@ CALCULATE_SITEWIDE_DAILY_METRICS = "CALCULATE_SITEWIDE_DAILY_METRICS" CALCULATE_SITEWIDE_ELECTION_METRICS = "CALCULATE_SITEWIDE_ELECTION_METRICS" CALCULATE_SITEWIDE_VOTER_METRICS = "CALCULATE_SITEWIDE_VOTER_METRICS" +GENERATE_VOTER_GUIDES = "GENERATE_VOTER_GUIDES" RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS = "RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS" REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS = "REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS" REFRESH_BALLOT_ITEMS_FROM_VOTERS = "REFRESH_BALLOT_ITEMS_FROM_VOTERS" @@ -620,6 +621,7 @@ (CALCULATE_SITEWIDE_ELECTION_METRICS, 'Sitewide election metrics'), (CALCULATE_ORGANIZATION_DAILY_METRICS, 'Organization specific daily metrics'), (CALCULATE_ORGANIZATION_ELECTION_METRICS, 'Organization specific election metrics'), + (GENERATE_VOTER_GUIDES, 'Generate voter guides'), (RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, 'Retrieve Ballot Items from Map Points'), (REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, 'Refresh Ballot Items from BallotReturned Map Points'), (REFRESH_BALLOT_ITEMS_FROM_VOTERS, 'Refresh Ballot Items from Voter Custom Addresses'), @@ -1414,7 +1416,7 @@ def fetch_batch_row_count(self, batch_header_id): """ try: - batch_row_query = BatchRow.objects.filter(batch_header_id=batch_header_id) + batch_row_query = BatchRow.objects.using('readonly').filter(batch_header_id=batch_header_id) batch_row_count = batch_row_query.count() except BatchRow.DoesNotExist: batch_row_count = 0 @@ -1434,47 +1436,56 @@ def fetch_batch_row_action_count(self, batch_header_id, kind_of_batch, kind_of_a batch_row_action_count = 0 try: if kind_of_batch == CANDIDATE: - batch_row_action_query = BatchRowActionCandidate.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionCandidate.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == CONTEST_OFFICE: - batch_row_action_query = BatchRowActionContestOffice.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionContestOffice.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == ELECTED_OFFICE: - batch_row_action_query = BatchRowActionElectedOffice.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionElectedOffice.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == IMPORT_BALLOT_ITEM: - batch_row_action_query = BatchRowActionBallotItem.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionBallotItem.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == IMPORT_POLLING_LOCATION: - batch_row_action_query = BatchRowActionPollingLocation.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionPollingLocation.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == MEASURE: - batch_row_action_query = BatchRowActionMeasure.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionMeasure.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == ORGANIZATION_WORD: - batch_row_action_query = BatchRowActionOrganization.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionOrganization.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == POLITICIAN: - batch_row_action_query = BatchRowActionPolitician.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionPolitician.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == POSITION: - batch_row_action_query = BatchRowActionPosition.objects.filter(batch_header_id=batch_header_id) + batch_row_action_query = BatchRowActionPosition.objects.using('readonly')\ + .filter(batch_header_id=batch_header_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() @@ -1494,47 +1505,56 @@ def fetch_batch_row_action_count_in_batch_set(self, batch_set_id, kind_of_batch, batch_row_action_count = 0 try: if kind_of_batch == CANDIDATE: - batch_row_action_query = BatchRowActionCandidate.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionCandidate.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == CONTEST_OFFICE: - batch_row_action_query = BatchRowActionContestOffice.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionContestOffice.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == ELECTED_OFFICE: - batch_row_action_query = BatchRowActionElectedOffice.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionElectedOffice.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == IMPORT_BALLOT_ITEM: - batch_row_action_query = BatchRowActionBallotItem.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionBallotItem.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == IMPORT_POLLING_LOCATION: - batch_row_action_query = BatchRowActionPollingLocation.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionPollingLocation.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == MEASURE: - batch_row_action_query = BatchRowActionMeasure.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionMeasure.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == ORGANIZATION_WORD: - batch_row_action_query = BatchRowActionOrganization.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionOrganization.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == POLITICIAN: - batch_row_action_query = BatchRowActionPolitician.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionPolitician.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() elif kind_of_batch == POSITION: - batch_row_action_query = BatchRowActionPosition.objects.filter(batch_set_id=batch_set_id) + batch_row_action_query = BatchRowActionPosition.objects.using('readonly')\ + .filter(batch_set_id=batch_set_id) if positive_value_exists(kind_of_action): batch_row_action_query = batch_row_action_query.filter(kind_of_action__iexact=kind_of_action) batch_row_action_count = batch_row_action_query.count() @@ -1582,12 +1602,17 @@ def retrieve_unprocessed_batch_set_info_by_election_and_set_source( } return results - def retrieve_batch_header_translation_suggestion(self, kind_of_batch, incoming_alternate_header_value): + def retrieve_batch_header_translation_suggestion( + self, + kind_of_batch, + incoming_alternate_header_value, + read_only=True): """ We are looking at one header value from a file imported by an admin or volunteer. We want to see if there are any suggestions for headers already recognized by We Vote. Ex/ "Organization" -> "organization_name" :param kind_of_batch: :param incoming_alternate_header_value: + :param read_only: :return: """ success = False @@ -1606,9 +1631,14 @@ def retrieve_batch_header_translation_suggestion(self, kind_of_batch, incoming_a try: # Note that we don't care about case sensitivity when we search for the alternate value - batch_header_translation_suggestion = BatchHeaderTranslationSuggestion.objects.get( - kind_of_batch=kind_of_batch, - incoming_alternate_header_value__iexact=incoming_alternate_header_value) + if positive_value_exists(read_only): + batch_header_translation_suggestion = BatchHeaderTranslationSuggestion.objects.using('readonly').get( + kind_of_batch=kind_of_batch, + incoming_alternate_header_value__iexact=incoming_alternate_header_value) + else: + batch_header_translation_suggestion = BatchHeaderTranslationSuggestion.objects.get( + kind_of_batch=kind_of_batch, + incoming_alternate_header_value__iexact=incoming_alternate_header_value) batch_header_translation_suggestion_found = True success = True status += "BATCH_HEADER_TRANSLATION_SUGGESTION_SAVED " @@ -1659,7 +1689,7 @@ def create_batch_row_translation_map( # TODO This hasn't been built } return results - def retrieve_batch_row_translation_map(self, kind_of_batch, incoming_alternate_header_value): + def retrieve_batch_row_translation_map(self, kind_of_batch, incoming_alternate_header_value, read_only=True): # TODO This hasn't been built yet success = False status = "" @@ -1677,9 +1707,14 @@ def retrieve_batch_row_translation_map(self, kind_of_batch, incoming_alternate_h try: # Note that we don't care about case sensitivity when we search for the alternate value - batch_header_translation_suggestion = BatchHeaderTranslationSuggestion.objects.get( - kind_of_batch=kind_of_batch, - incoming_alternate_header_value__iexact=incoming_alternate_header_value) + if positive_value_exists(read_only): + batch_header_translation_suggestion = BatchHeaderTranslationSuggestion.objects.using('read_only').get( + kind_of_batch=kind_of_batch, + incoming_alternate_header_value__iexact=incoming_alternate_header_value) + else: + batch_header_translation_suggestion = BatchHeaderTranslationSuggestion.objects.get( + kind_of_batch=kind_of_batch, + incoming_alternate_header_value__iexact=incoming_alternate_header_value) batch_header_translation_suggestion_found = True success = True status += "BATCH_HEADER_TRANSLATION_SUGGESTION_SAVED " @@ -3669,27 +3704,31 @@ def count_number_of_batch_action_rows(self, header_id, kind_of_batch): number_of_batch_action_rows = 0 if positive_value_exists(header_id): if kind_of_batch == MEASURE: - number_of_batch_action_rows = BatchRowActionMeasure.objects.filter(batch_header_id=header_id).count() + number_of_batch_action_rows = BatchRowActionMeasure.objects.using('readonly')\ + .filter(batch_header_id=header_id).count() elif kind_of_batch == ELECTED_OFFICE: - number_of_batch_action_rows = BatchRowActionElectedOffice.objects.filter(batch_header_id=header_id).\ - count() + number_of_batch_action_rows = BatchRowActionElectedOffice.objects.using('readonly')\ + .filter(batch_header_id=header_id).count() elif kind_of_batch == CONTEST_OFFICE: - number_of_batch_action_rows = BatchRowActionContestOffice.objects.filter(batch_header_id=header_id).\ - count() + number_of_batch_action_rows = BatchRowActionContestOffice.objects.using('readonly')\ + .filter(batch_header_id=header_id).count() elif kind_of_batch == CANDIDATE: - number_of_batch_action_rows = BatchRowActionCandidate.objects.filter(batch_header_id=header_id).count() + number_of_batch_action_rows = BatchRowActionCandidate.objects.using('readonly')\ + .filter(batch_header_id=header_id).count() elif kind_of_batch == POLITICIAN: - number_of_batch_action_rows = BatchRowActionPolitician.objects.filter(batch_header_id=header_id).count() + number_of_batch_action_rows = BatchRowActionPolitician.objects.using('readonly')\ + .filter(batch_header_id=header_id).count() else: number_of_batch_action_rows = 0 return number_of_batch_action_rows def count_number_of_batches_in_batch_set(self, batch_set_id=0, batch_row_analyzed=None, batch_row_created=None): number_of_batches = 0 - batch_description_query = BatchDescription.objects.filter(batch_set_id=batch_set_id) + batch_description_query = BatchDescription.objects.using('readonly').filter(batch_set_id=batch_set_id) batch_description_list = list(batch_description_query) for batch_description in batch_description_list: - batch_row_query = BatchRow.objects.filter(batch_header_id=batch_description.batch_header_id) + batch_row_query = BatchRow.objects.using('readonly')\ + .filter(batch_header_id=batch_description.batch_header_id) if batch_row_analyzed is not None: batch_row_query = batch_row_query.filter(batch_row_analyzed=batch_row_analyzed) if batch_row_created is not None: @@ -3707,7 +3746,10 @@ def fetch_batch_header_translation_suggestion(self, kind_of_batch, alternate_hea :param alternate_header_value: :return: """ - results = self.retrieve_batch_header_translation_suggestion(kind_of_batch, alternate_header_value) + results = self.retrieve_batch_header_translation_suggestion( + kind_of_batch, + alternate_header_value, + read_only=True) if results['batch_header_translation_suggestion_found']: batch_header_translation_suggestion = results['batch_header_translation_suggestion'] return batch_header_translation_suggestion.header_value_recognized_by_we_vote @@ -3715,7 +3757,7 @@ def fetch_batch_header_translation_suggestion(self, kind_of_batch, alternate_hea # TODO This hasn't been built def fetch_batch_row_translation_map(self, kind_of_batch, batch_row_name, incoming_alternate_row_value): - results = self.retrieve_batch_row_translation_map(kind_of_batch, incoming_alternate_row_value) + results = self.retrieve_batch_row_translation_map(kind_of_batch, incoming_alternate_row_value, read_only=True) if results['batch_header_translation_suggestion_found']: batch_header_translation_suggestion = results['batch_header_translation_suggestion'] return batch_header_translation_suggestion.header_value_recognized_by_we_vote @@ -3734,8 +3776,9 @@ def fetch_elected_office_name_from_elected_office_ctcl_id(self, elected_office_c # batch_header_id = get_batch_header_id_from_batch_description(batch_set_id, ELECTED_OFFICE) try: if positive_value_exists(batch_set_id): - batch_description_on_stage = BatchDescription.objects.get(batch_set_id=batch_set_id, - kind_of_batch=ELECTED_OFFICE) + batch_description_on_stage = BatchDescription.objects.using('readonly').get( + batch_set_id=batch_set_id, + kind_of_batch=ELECTED_OFFICE) if batch_description_on_stage: batch_header_id = batch_description_on_stage.batch_header_id except BatchDescription.DoesNotExist: @@ -3747,7 +3790,7 @@ def fetch_elected_office_name_from_elected_office_ctcl_id(self, elected_office_c try: batch_manager = BatchManager() if positive_value_exists(batch_header_id) and elected_office_ctcl_id: - batch_header_map = BatchHeaderMap.objects.get(batch_header_id=batch_header_id) + batch_header_map = BatchHeaderMap.objects.using('readonly').get(batch_header_id=batch_header_id) # Get the column name in BatchRow that stores elected_office_batch_id - id taken from batch_header_map # eg: batch_row_000 -> elected_office_batch_id @@ -3756,8 +3799,9 @@ def fetch_elected_office_name_from_elected_office_ctcl_id(self, elected_office_c # we found batch row column name corresponding to elected_office_batch_id, now look up batch_row table # with given batch_header_id and elected_office_batch_id (batch_row_00) - batch_row_on_stage = BatchRow.objects.get(batch_header_id=batch_header_id, - **{ elected_office_id_column_name: elected_office_ctcl_id}) + batch_row_on_stage = BatchRow.objects.using('readonly').get( + batch_header_id=batch_header_id, + **{elected_office_id_column_name: elected_office_ctcl_id}) # we know the batch row, next retrieve value for elected_office_name eg: off1 -> NC State Senator elected_office_name = batch_manager.retrieve_value_from_batch_row('elected_office_name', batch_header_map, batch_row_on_stage) @@ -3780,8 +3824,9 @@ def fetch_state_code_from_person_id_in_candidate(self, person_id, batch_set_id): # batch_header_id = get_batch_header_id_from_batch_description(batch_set_id, CANDIDATE) try: if positive_value_exists(batch_set_id): - batch_description_on_stage = BatchDescription.objects.get(batch_set_id=batch_set_id, - kind_of_batch=CANDIDATE) + batch_description_on_stage = BatchDescription.objects.using('readonly').get( + batch_set_id=batch_set_id, + kind_of_batch=CANDIDATE) if batch_description_on_stage: batch_header_id = batch_description_on_stage.batch_header_id except BatchDescription.DoesNotExist: @@ -3789,8 +3834,9 @@ def fetch_state_code_from_person_id_in_candidate(self, person_id, batch_set_id): try: if positive_value_exists(batch_header_id) and person_id is not None: - batchrowaction_candidate = BatchRowActionCandidate.objects.get(batch_header_id=batch_header_id, - candidate_ctcl_person_id=person_id) + batchrowaction_candidate = BatchRowActionCandidate.objects.using('readonly').get( + batch_header_id=batch_header_id, + candidate_ctcl_person_id=person_id) if batchrowaction_candidate is not None: state_code = batchrowaction_candidate.state_code @@ -4916,6 +4962,7 @@ def create_batch_process( CALCULATE_SITEWIDE_ELECTION_METRICS, CALCULATE_ORGANIZATION_DAILY_METRICS, CALCULATE_ORGANIZATION_ELECTION_METRICS, + GENERATE_VOTER_GUIDES, REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, REFRESH_BALLOT_ITEMS_FROM_VOTERS, RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, @@ -5088,14 +5135,14 @@ def count_active_batch_processes(self): batch_process_count = 0 election_manager = ElectionManager() - results = election_manager.retrieve_upcoming_elections() + results = election_manager.retrieve_upcoming_elections(read_only=True) election_list = results['election_list'] google_civic_election_id_list = [] for one_election in election_list: google_civic_election_id_list.append(one_election.google_civic_election_id) try: - batch_process_queryset = BatchProcess.objects.all() + batch_process_queryset = BatchProcess.objects.using('readonly').all() batch_process_queryset = batch_process_queryset.filter(date_started__isnull=False) batch_process_queryset = batch_process_queryset.filter(date_completed__isnull=True) batch_process_queryset = batch_process_queryset.exclude(batch_process_paused=True) @@ -5112,14 +5159,14 @@ def count_checked_out_batch_processes(self): batch_process_count = 0 election_manager = ElectionManager() - results = election_manager.retrieve_upcoming_elections() + results = election_manager.retrieve_upcoming_elections(read_only=True) election_list = results['election_list'] google_civic_election_id_list = [] for one_election in election_list: google_civic_election_id_list.append(one_election.google_civic_election_id) try: - batch_process_queryset = BatchProcess.objects.all() + batch_process_queryset = BatchProcess.objects.using('readonly').all() batch_process_queryset = batch_process_queryset.filter(date_started__isnull=False) batch_process_queryset = batch_process_queryset.filter(date_completed__isnull=True) batch_process_queryset = batch_process_queryset.filter(date_checked_out__isnull=False) @@ -5140,6 +5187,7 @@ def count_checked_out_batch_processes(self): # CALCULATE_SITEWIDE_ELECTION_METRICS # CALCULATE_ORGANIZATION_DAILY_METRICS # CALCULATE_ORGANIZATION_ELECTION_METRICS + # GENERATE_VOTER_GUIDES # REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS # REFRESH_BALLOT_ITEMS_FROM_VOTERS # RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS @@ -5161,14 +5209,14 @@ def count_next_steps( RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS in kind_of_process_list if related_to_upcoming_election: election_manager = ElectionManager() - results = election_manager.retrieve_upcoming_elections() + results = election_manager.retrieve_upcoming_elections(read_only=True) election_list = results['election_list'] google_civic_election_id_list = [] for one_election in election_list: google_civic_election_id_list.append(one_election.google_civic_election_id) try: - batch_process_queryset = BatchProcess.objects.all() + batch_process_queryset = BatchProcess.objects.using('readonly').all() batch_process_queryset = batch_process_queryset.filter(kind_of_process__in=kind_of_process_list) batch_process_queryset = batch_process_queryset.exclude(batch_process_paused=True) if positive_value_exists(is_active): @@ -5197,7 +5245,7 @@ def is_batch_process_currently_scheduled( self, google_civic_election_id=0, state_code="", kind_of_process=""): status = "" try: - batch_process_queryset = BatchProcess.objects.all() + batch_process_queryset = BatchProcess.objects.using('readonly').all() batch_process_queryset = batch_process_queryset.filter(google_civic_election_id=google_civic_election_id) batch_process_queryset = batch_process_queryset.filter(state_code=state_code) batch_process_queryset = batch_process_queryset.filter(kind_of_process=kind_of_process) @@ -5214,7 +5262,7 @@ def is_activity_notice_process_currently_running(self): status = "" analytics_kind_of_process_list = [ACTIVITY_NOTICE_PROCESS] try: - batch_process_queryset = BatchProcess.objects.all() + batch_process_queryset = BatchProcess.objects.using('readonly').all() batch_process_queryset = batch_process_queryset.filter(date_started__isnull=False) batch_process_queryset = batch_process_queryset.filter(date_completed__isnull=True) batch_process_queryset = batch_process_queryset.filter(date_checked_out__isnull=False) @@ -5238,7 +5286,7 @@ def is_analytics_process_currently_running(self): CALCULATE_SITEWIDE_ELECTION_METRICS, CALCULATE_ORGANIZATION_DAILY_METRICS, CALCULATE_ORGANIZATION_ELECTION_METRICS] try: - batch_process_queryset = BatchProcess.objects.all() + batch_process_queryset = BatchProcess.objects.using('readonly').all() batch_process_queryset = batch_process_queryset.filter(date_started__isnull=False) batch_process_queryset = batch_process_queryset.filter(date_completed__isnull=True) batch_process_queryset = batch_process_queryset.filter(date_checked_out__isnull=False) @@ -5316,6 +5364,8 @@ def retrieve_batch_process_list( checked_out_expiration_time = 270 # 4.5 minutes * 60 seconds elif batch_process.kind_of_process == API_REFRESH_REQUEST: checked_out_expiration_time = 360 # 6 minutes * 60 seconds + elif batch_process.kind_of_process == GENERATE_VOTER_GUIDES: + checked_out_expiration_time = 600 # 10 minutes * 60 seconds elif batch_process.kind_of_process in [ REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, REFRESH_BALLOT_ITEMS_FROM_VOTERS, RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS]: @@ -5468,7 +5518,7 @@ class BatchProcess(models.Model): """ """ kind_of_process = models.CharField(max_length=50, choices=KIND_OF_PROCESS_CHOICES, - default=RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS) + default=RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, db_index=True) # The unique ID of this election. (Provided by Google Civic) google_civic_election_id = models.PositiveIntegerField( @@ -5490,13 +5540,13 @@ class BatchProcess(models.Model): election_id_list_serialized = models.CharField(max_length=255, null=True) date_added_to_queue = models.DateTimeField(verbose_name='start', null=True) - date_started = models.DateTimeField(verbose_name='start', null=True) + date_started = models.DateTimeField(verbose_name='start', null=True, db_index=True) # When have all of the steps completed? - date_completed = models.DateTimeField(verbose_name='finished', null=True) + date_completed = models.DateTimeField(verbose_name='finished', null=True, db_index=True) # When a batch_process is running, we mark when it was "taken off the shelf" to be worked on. # When the process is complete, we should reset this to "NULL" date_checked_out = models.DateTimeField(null=True) - batch_process_paused = models.BooleanField(default=False) + batch_process_paused = models.BooleanField(default=False, db_index=True) completion_summary = models.TextField(null=True, blank=True) use_ballotpedia = models.BooleanField(default=False) use_ctcl = models.BooleanField(default=False) diff --git a/import_export_batches/views_admin.py b/import_export_batches/views_admin.py index 619102a3f..c461e4cf0 100755 --- a/import_export_batches/views_admin.py +++ b/import_export_batches/views_admin.py @@ -43,7 +43,8 @@ from import_export_vote_usa.controllers import VOTE_USA_VOTER_INFO_URL import json import math -from polling_location.models import KIND_OF_LOG_ENTRY_BALLOT_RECEIVED, PollingLocation, PollingLocationManager +from polling_location.models import KIND_OF_LOG_ENTRY_BALLOT_RECEIVED, MAP_POINTS_RETRIEVED_EACH_BATCH_CHUNK,\ + PollingLocation, PollingLocationManager from position.models import POSITION import random import requests @@ -52,8 +53,6 @@ import wevote_functions.admin from wevote_functions.functions import convert_to_int, positive_value_exists, STATE_CODE_MAP -MAP_POINTS_RETRIEVED_EACH_BATCH_CHUNK = 125 # 125. Formerly 250 and 111 - logger = wevote_functions.admin.get_logger(__name__) @@ -390,6 +389,7 @@ def batch_action_list_view(request): batch_set_list = [] polling_location_we_vote_id = "" + status = "" batch_header_id = convert_to_int(request.GET.get('batch_header_id', 0)) kind_of_batch = request.GET.get('kind_of_batch', '') @@ -563,6 +563,7 @@ def batch_action_list_view(request): modified_batch_row_list.append(one_batch_row) elif kind_of_batch == POSITION: existing_results = batch_manager.retrieve_batch_row_action_position(batch_header_id, one_batch_row.id) + status += existing_results['status'] if existing_results['batch_row_action_found']: one_batch_row.batch_row_action = existing_results['batch_row_action_position'] one_batch_row.kind_of_batch = POSITION @@ -599,8 +600,8 @@ def batch_action_list_view(request): if one_state[0].lower() in active_state_codes: filtered_state_list.append(one_state) - messages.add_message(request, messages.INFO, 'Batch Row Count: {batch_row_count}' - ''.format(batch_row_count=batch_row_count)) + messages.add_message(request, messages.INFO, 'Batch Row Count: {batch_row_count}, status: {status}' + ''.format(batch_row_count=batch_row_count, status=status)) messages_on_stage = get_messages(request) @@ -900,7 +901,7 @@ def batch_action_list_export_voters_view(request): @login_required def batch_action_list_analyze_process_view(request): """ - Create BatchRowActions for either all of the BatchRows for batch_header_id, or only one with batch_row_id + Create BatchRowActions for either all the BatchRows for batch_header_id, or only one with batch_row_id :param request: :return: """ @@ -1523,6 +1524,8 @@ def batch_process_system_toggle_view(request): setting_name = 'batch_process_system_ballot_items_on' elif kind_of_process == 'CALCULATE_ANALYTICS': setting_name = 'batch_process_system_calculate_analytics_on' + elif kind_of_process == 'GENERATE_VOTER_GUIDES': + setting_name = 'batch_process_system_generate_voter_guides_on' elif kind_of_process == 'SEARCH_TWITTER': setting_name = 'batch_process_system_search_twitter_on' elif kind_of_process == 'UPDATE_TWITTER_DATA': @@ -1684,6 +1687,9 @@ def batch_process_list_view(request): 'REFRESH_BALLOT_ITEMS_FROM_VOTERS', 'RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS'] batch_process_queryset = batch_process_queryset.filter(kind_of_process__in=ballot_item_processes) + elif kind_of_processes_to_show == "GENERATE_VOTER_GUIDES": + processes = ['GENERATE_VOTER_GUIDES'] + batch_process_queryset = batch_process_queryset.filter(kind_of_process__in=processes) elif kind_of_processes_to_show == "SEARCH_TWITTER": search_twitter_processes = ['SEARCH_TWITTER_FOR_CANDIDATE_TWITTER_HANDLE'] batch_process_queryset = batch_process_queryset.filter(kind_of_process__in=search_twitter_processes) @@ -1747,23 +1753,29 @@ def batch_process_list_view(request): state_codes_map_point_counts_dict = {} polling_location_manager = PollingLocationManager() + map_points_retrieved_each_batch_chunk = 102 # Signals that a batch_process wasn't found for batch_process in batch_process_list: if batch_process.kind_of_process in [ RETRIEVE_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS, ]: state_code_lower_case = '' + map_points_retrieved_each_batch_chunk = 101 # Signals that a state_code wasn't found if positive_value_exists(batch_process.state_code): state_code_lower_case = batch_process.state_code.lower() + # For both REFRESH and RETRIEVE, see if number of map points for this state exceed the "large" threshold + map_points_retrieved_each_batch_chunk = \ + polling_location_manager.calculate_number_of_map_points_to_retrieve_with_each_batch_chunk( + state_code_lower_case) if state_code_lower_case in state_codes_map_point_counts_dict: batch_process.polling_location_count = state_codes_map_point_counts_dict[state_code_lower_case] batch_process.ballot_item_chunks_expected = \ - int(math.ceil(batch_process.polling_location_count / 125)) + 1 + int(math.ceil(batch_process.polling_location_count / map_points_retrieved_each_batch_chunk)) + 1 else: state_codes_map_point_counts_dict[state_code_lower_case] = \ polling_location_manager.fetch_polling_location_count(state_code=state_code_lower_case) batch_process.polling_location_count = state_codes_map_point_counts_dict[state_code_lower_case] batch_process.ballot_item_chunks_expected = \ - int(math.ceil(batch_process.polling_location_count / 125)) + 1 + int(math.ceil(batch_process.polling_location_count / map_points_retrieved_each_batch_chunk)) + 1 # Add the processing "chunks" under each Batch Process batch_process_ballot_item_chunk_list = [] batch_process_ballot_item_chunk_list_found = False @@ -1857,13 +1869,15 @@ def batch_process_list_view(request): from wevote_settings.models import fetch_batch_process_system_on, fetch_batch_process_system_activity_notices_on, \ fetch_batch_process_system_api_refresh_on, fetch_batch_process_system_ballot_items_on, \ - fetch_batch_process_system_calculate_analytics_on, fetch_batch_process_system_search_twitter_on, \ + fetch_batch_process_system_calculate_analytics_on, fetch_batch_process_system_generate_voter_guides_on, \ + fetch_batch_process_system_search_twitter_on, \ fetch_batch_process_system_update_twitter_on batch_process_system_on = fetch_batch_process_system_on() batch_process_system_activity_notices_on = fetch_batch_process_system_activity_notices_on() batch_process_system_api_refresh_on = fetch_batch_process_system_api_refresh_on() batch_process_system_ballot_items_on = fetch_batch_process_system_ballot_items_on() batch_process_system_calculate_analytics_on = fetch_batch_process_system_calculate_analytics_on() + batch_process_system_generate_voter_guides_on = fetch_batch_process_system_generate_voter_guides_on() batch_process_system_search_twitter_on = fetch_batch_process_system_search_twitter_on() batch_process_system_update_twitter_on = fetch_batch_process_system_update_twitter_on() @@ -1896,7 +1910,6 @@ def batch_process_list_view(request): toggle_system_url_variables += "&show_paused_processes_only=1" if positive_value_exists(state_code): toggle_system_url_variables += "&state_code=" + str(state_code) - template_values = { 'messages_on_stage': messages_on_stage, 'ballot_returned_oldest_date': ballot_returned_oldest_date, @@ -1908,6 +1921,7 @@ def batch_process_list_view(request): 'batch_process_system_api_refresh_on': batch_process_system_api_refresh_on, 'batch_process_system_ballot_items_on': batch_process_system_ballot_items_on, 'batch_process_system_calculate_analytics_on': batch_process_system_calculate_analytics_on, + 'batch_process_system_generate_voter_guides_on': batch_process_system_generate_voter_guides_on, 'batch_process_system_search_twitter_on': batch_process_system_search_twitter_on, 'batch_process_system_update_twitter_on': batch_process_system_update_twitter_on, 'batch_process_search': batch_process_search, @@ -1915,6 +1929,7 @@ def batch_process_list_view(request): 'google_civic_election_id': google_civic_election_id, 'include_frequent_processes': include_frequent_processes, 'kind_of_processes_to_show': kind_of_processes_to_show, + 'map_points_retrieved_each_batch_chunk': map_points_retrieved_each_batch_chunk, 'show_all_elections': show_all_elections, 'show_active_processes_only': show_active_processes_only, 'show_paused_processes_only': show_paused_processes_only, @@ -3132,7 +3147,7 @@ def retrieve_ballots_for_polling_locations_api_v4_view(request): from lat/long, and then the ballot items. Ballotpedia API v4 Reach out to Ballotpedia and retrieve (for one election): 1) Polling locations (so we can use those addresses to retrieve a representative set of ballots) - 2) Cycle through a portion of those map points, enough that we are caching all of the possible ballot items + 2) Cycle through a portion of those map points, enough that we are caching all the possible ballot items :param request: :return: """ @@ -3342,6 +3357,7 @@ def retrieve_ballots_for_polling_locations_api_v4_internal_view( } return results + polling_location_manager = PollingLocationManager() try: if positive_value_exists(refresh_ballot_returned): kind_of_process = REFRESH_BALLOT_ITEMS_FROM_POLLING_LOCATIONS @@ -3401,7 +3417,6 @@ def retrieve_ballots_for_polling_locations_api_v4_internal_view( # Find polling_location_we_vote_ids already used in this batch_process, which returned a ballot polling_location_we_vote_id_list_already_retrieved = [] if positive_value_exists(batch_process_id): - polling_location_manager = PollingLocationManager() polling_location_log_entry_list = polling_location_manager.retrieve_polling_location_log_entry_list( batch_process_id=batch_process_id, is_from_ctcl=use_ctcl, @@ -3412,7 +3427,7 @@ def retrieve_ballots_for_polling_locations_api_v4_internal_view( if one_log_entry.polling_location_we_vote_id not in polling_location_we_vote_id_list_already_retrieved: polling_location_we_vote_id_list_already_retrieved.append(one_log_entry.polling_location_we_vote_id) - # For both REFRESH and RETRIEVE, find polling locations/map points which have came up empty + # For both REFRESH and RETRIEVE, find polling locations/map points which have come up empty # (from this data source) in previous chunks since when this process started polling_location_we_vote_id_list_returned_empty = [] results = ballot_returned_list_manager.\ @@ -3427,7 +3442,11 @@ def retrieve_ballots_for_polling_locations_api_v4_internal_view( polling_location_we_vote_id_list_returned_empty = results['polling_location_we_vote_id_list'] status += "REFRESH_BALLOT_RETURNED: " + str(refresh_ballot_returned) + " " - refresh_or_retrieve_limit = MAP_POINTS_RETRIEVED_EACH_BATCH_CHUNK # 125. Formerly 250 and 111 + + # For both REFRESH and RETRIEVE, see if the number of map points for this state exceed the "large" threshold + refresh_or_retrieve_limit = \ + polling_location_manager.calculate_number_of_map_points_to_retrieve_with_each_batch_chunk(state_code) + if positive_value_exists(refresh_ballot_returned): # REFRESH branch polling_location_query = PollingLocation.objects.using('readonly').all() @@ -3443,6 +3462,10 @@ def retrieve_ballots_for_polling_locations_api_v4_internal_view( polling_location_we_vote_id_list_to_retrieve[:refresh_or_retrieve_limit] polling_location_query = \ polling_location_query.filter(we_vote_id__in=polling_location_we_vote_id_list_to_retrieve_limited) + if positive_value_exists(use_ctcl): + # CTCL only supports full addresses, so don't bother trying to pass addresses without line1 + polling_location_query = \ + polling_location_query.exclude(Q(line1__isnull=True) | Q(line1__exact='')) # We don't exclude the deleted map points because we need to know to delete the ballot returned entry # polling_location_query = polling_location_query.exclude(polling_location_deleted=True) polling_location_list = list(polling_location_query) @@ -3464,8 +3487,12 @@ def retrieve_ballots_for_polling_locations_api_v4_internal_view( polling_location_query = \ polling_location_query.exclude(we_vote_id__in=polling_location_we_vote_id_list_to_exclude) polling_location_query = polling_location_query.exclude(polling_location_deleted=True) + if positive_value_exists(use_ctcl): + # CTCL only supports full addresses, so don't bother trying to pass addresses without line1 + polling_location_query = \ + polling_location_query.exclude(Q(line1__isnull=True) | Q(line1__exact='')) - # Randomly change the sort order so we over time load different map points (before timeout) + # Randomly change the sort order, so we over time load different map points (before timeout) random_sorting = random.randint(1, 5) if random_sorting == 1: # Ordering by "line1" creates a bit of (locational) random order diff --git a/import_export_facebook/controllers.py b/import_export_facebook/controllers.py index 8b91746f4..ad27f5bd3 100644 --- a/import_export_facebook/controllers.py +++ b/import_export_facebook/controllers.py @@ -54,7 +54,8 @@ def voter_facebook_save_to_current_account_for_api(voter_device_id): # voterFac voter = results['voter'] facebook_manager = FacebookManager() - facebook_results = facebook_manager.retrieve_facebook_link_to_voter(voter_we_vote_id=voter.we_vote_id) + facebook_results = facebook_manager.retrieve_facebook_link_to_voter( + voter_we_vote_id=voter.we_vote_id, read_only=True) if facebook_results['facebook_link_to_voter_found']: error_results = { 'status': "FACEBOOK_OWNER_VOTER_FOUND_WHEN_NOT_EXPECTED", @@ -275,14 +276,14 @@ def facebook_friends_action_for_api(voter_device_id): # facebookFriendsAction # Find facebook_link_to_voter for all users and then updating SuggestedFriend table facebook_auth_response = auth_response_results['facebook_auth_response'] my_facebook_link_to_voter_results = facebook_manager.retrieve_facebook_link_to_voter( - facebook_auth_response.facebook_user_id) + facebook_auth_response.facebook_user_id, read_only=True) status += ' ' + my_facebook_link_to_voter_results['status'] if my_facebook_link_to_voter_results['facebook_link_to_voter_found']: friend_manager = FriendManager() viewer_voter_we_vote_id = my_facebook_link_to_voter_results['facebook_link_to_voter'].voter_we_vote_id for facebook_user_entry in facebook_friends_using_we_vote_list: facebook_user_link_to_voter_results = facebook_manager.retrieve_facebook_link_to_voter( - facebook_user_entry['facebook_user_id']) + facebook_user_entry['facebook_user_id'], read_only=True) status += ' ' + facebook_user_link_to_voter_results['status'] if facebook_user_link_to_voter_results['facebook_link_to_voter_found']: viewee_voter_we_vote_id = facebook_user_link_to_voter_results['facebook_link_to_voter'].voter_we_vote_id @@ -474,7 +475,8 @@ def voter_facebook_sign_in_retrieve_for_api(voter_device_id): # voterFacebookSi voter_manager = VoterManager() organization_manager = OrganizationManager() - facebook_link_results = facebook_manager.retrieve_facebook_link_to_voter(facebook_auth_response.facebook_user_id) + facebook_link_results = facebook_manager.retrieve_facebook_link_to_voter( + facebook_auth_response.facebook_user_id, read_only=True) if facebook_link_results['facebook_link_to_voter_found']: status += "FACEBOOK_SIGN_IN_RETRIEVE-FACEBOOK_LINK_TO_VOTER_FOUND " facebook_link_to_voter = facebook_link_results['facebook_link_to_voter'] @@ -502,7 +504,7 @@ def voter_facebook_sign_in_retrieve_for_api(voter_device_id): # voterFacebookSi facebook_auth_response.facebook_user_id, voter_we_vote_id_attached_to_facebook) status += " " + save_results['status'] facebook_link_results = facebook_manager.retrieve_facebook_link_to_voter( - facebook_auth_response.facebook_user_id) + facebook_auth_response.facebook_user_id, read_only=True) if facebook_link_results['facebook_link_to_voter_found']: facebook_link_to_voter = facebook_link_results['facebook_link_to_voter'] repair_facebook_related_voter_caching_now = True diff --git a/import_export_facebook/models.py b/import_export_facebook/models.py index f552a8d38..5ecdeab91 100644 --- a/import_export_facebook/models.py +++ b/import_export_facebook/models.py @@ -689,31 +689,40 @@ def retrieve_facebook_auth_response_from_facebook_id(self, facebook_user_id): def fetch_facebook_id_from_voter_we_vote_id(self, voter_we_vote_id): facebook_user_id = 0 - facebook_results = self.retrieve_facebook_link_to_voter(facebook_user_id, voter_we_vote_id) + facebook_results = self.retrieve_facebook_link_to_voter(facebook_user_id, voter_we_vote_id, read_only=True) if facebook_results['facebook_link_to_voter_found']: facebook_link_to_voter = facebook_results['facebook_link_to_voter'] facebook_user_id = facebook_link_to_voter.facebook_user_id return facebook_user_id - def retrieve_facebook_link_to_voter_from_facebook_id(self, facebook_user_id): - return self.retrieve_facebook_link_to_voter(facebook_user_id) + def retrieve_facebook_link_to_voter_from_facebook_id(self, facebook_user_id, read_only=False): + return self.retrieve_facebook_link_to_voter(facebook_user_id, read_only=read_only) - def retrieve_facebook_link_to_voter_from_voter_we_vote_id(self, voter_we_vote_id): + def retrieve_facebook_link_to_voter_from_voter_we_vote_id(self, voter_we_vote_id, read_only=False): facebook_user_id = 0 facebook_secret_key = "" - return self.retrieve_facebook_link_to_voter(facebook_user_id, voter_we_vote_id, facebook_secret_key) + return self.retrieve_facebook_link_to_voter( + facebook_user_id, voter_we_vote_id, facebook_secret_key, read_only=read_only) - def retrieve_facebook_link_to_voter_from_facebook_secret_key(self, facebook_secret_key): + def retrieve_facebook_link_to_voter_from_facebook_secret_key(self, facebook_secret_key, read_only=False): facebook_user_id = 0 voter_we_vote_id = "" - return self.retrieve_facebook_link_to_voter(facebook_user_id, voter_we_vote_id, facebook_secret_key) - - def retrieve_facebook_link_to_voter(self, facebook_user_id=0, voter_we_vote_id='', facebook_secret_key=''): + return self.retrieve_facebook_link_to_voter( + facebook_user_id, voter_we_vote_id, facebook_secret_key, read_only=read_only) + + def retrieve_facebook_link_to_voter( + self, + facebook_user_id=0, + voter_we_vote_id='', + facebook_secret_key='', + read_only=False, + ): """ :param facebook_user_id: :param voter_we_vote_id: :param facebook_secret_key: + :param read_only: :return: """ facebook_link_to_voter = FacebookLinkToVoter() @@ -721,25 +730,40 @@ def retrieve_facebook_link_to_voter(self, facebook_user_id=0, voter_we_vote_id=' try: if positive_value_exists(facebook_user_id): - facebook_link_to_voter = FacebookLinkToVoter.objects.get( - facebook_user_id=facebook_user_id, - ) + if positive_value_exists(read_only): + facebook_link_to_voter = FacebookLinkToVoter.objects.using('readonly').get( + facebook_user_id=facebook_user_id, + ) + else: + facebook_link_to_voter = FacebookLinkToVoter.objects.get( + facebook_user_id=facebook_user_id, + ) facebook_link_to_voter_id = facebook_link_to_voter.id facebook_link_to_voter_found = True success = True status = "RETRIEVE_FACEBOOK_LINK_TO_VOTER_FOUND_BY_FACEBOOK_USER_ID " elif positive_value_exists(voter_we_vote_id): - facebook_link_to_voter = FacebookLinkToVoter.objects.get( - voter_we_vote_id__iexact=voter_we_vote_id, - ) + if positive_value_exists(read_only): + facebook_link_to_voter = FacebookLinkToVoter.objects.using('readonly').get( + voter_we_vote_id__iexact=voter_we_vote_id, + ) + else: + facebook_link_to_voter = FacebookLinkToVoter.objects.get( + voter_we_vote_id__iexact=voter_we_vote_id, + ) facebook_link_to_voter_id = facebook_link_to_voter.id facebook_link_to_voter_found = True success = True status = "RETRIEVE_FACEBOOK_LINK_TO_VOTER_FOUND_BY_VOTER_WE_VOTE_ID " elif positive_value_exists(facebook_secret_key): - facebook_link_to_voter = FacebookLinkToVoter.objects.get( - secret_key=facebook_secret_key, - ) + if positive_value_exists(read_only): + facebook_link_to_voter = FacebookLinkToVoter.objects.using('readonly').get( + secret_key=facebook_secret_key, + ) + else: + facebook_link_to_voter = FacebookLinkToVoter.objects.get( + secret_key=facebook_secret_key, + ) facebook_link_to_voter_id = facebook_link_to_voter.id facebook_link_to_voter_found = True success = True diff --git a/import_export_google_civic/controllers.py b/import_export_google_civic/controllers.py index 56c6e8342..d476a1fe3 100644 --- a/import_export_google_civic/controllers.py +++ b/import_export_google_civic/controllers.py @@ -25,7 +25,7 @@ from wevote_functions.functions import convert_district_scope_to_ballotpedia_race_office_level, \ convert_state_text_to_state_code, convert_to_int, \ extract_district_id_label_when_district_id_exists_from_ocd_id, extract_district_id_from_ocd_division_id, \ - extract_facebook_username_from_text_string, \ + extract_facebook_username_from_text_string, extract_instagram_handle_from_text_string, \ extract_state_code_from_address_string, extract_state_from_ocd_division_id, \ extract_twitter_handle_from_text_string, extract_vote_usa_measure_id, extract_vote_usa_office_id, \ is_voter_device_id_valid, logger, positive_value_exists, STATE_CODE_MAP @@ -562,6 +562,8 @@ def groom_and_store_google_civic_candidates_json_2021( go_fund_me_url = 'https://' + go_fund_me_url if one_channel['type'] == 'Instagram': instagram_handle = one_channel['id'] if 'id' in one_channel else '' + if positive_value_exists(instagram_handle): + instagram_handle = extract_instagram_handle_from_text_string(instagram_handle) if one_channel['type'] == 'LinkedIn': linkedin_url = one_channel['id'] if 'id' in one_channel else '' if positive_value_exists(linkedin_url): @@ -713,7 +715,7 @@ def groom_and_store_google_civic_candidates_json_2021( candidate_twitter_handle=None, candidate_name=candidate_name, instagram_handle=None, - read_only=True) + read_only=False) if not results['success']: continue_searching_for_candidate = False status += "FAILED_RETRIEVING_CANDIDATE_FROM_UNIQUE_IDS: " + results['status'] + " " diff --git a/import_export_google_civic/models.py b/import_export_google_civic/models.py index be2328bce..0e553fc83 100644 --- a/import_export_google_civic/models.py +++ b/import_export_google_civic/models.py @@ -23,7 +23,7 @@ def retrieve_google_civic_election_id_for_voter(voter_id): if positive_value_exists(voter_id): try: - ballot_item_query = BallotItem.objects.filter( + ballot_item_query = BallotItem.objects.using('readonly').filter( voter_id__exact=voter_id, ) ballot_item_list = list(ballot_item_query[:1]) diff --git a/import_export_twitter/controllers.py b/import_export_twitter/controllers.py index 77bfa95c5..c9ce635e5 100644 --- a/import_export_twitter/controllers.py +++ b/import_export_twitter/controllers.py @@ -247,8 +247,9 @@ def fetch_number_of_candidates_needing_twitter_search(): Q(candidate_twitter_handle__isnull=True) | Q(candidate_twitter_handle="")) # Exclude candidates we have already have TwitterLinkPossibility data for try: - twitter_possibility_list = TwitterLinkPossibility.objects.using('readonly'). \ + twitter_possibility_query = TwitterLinkPossibility.objects.using('readonly'). \ values_list('candidate_campaign_we_vote_id', flat=True).distinct() + twitter_possibility_list = list(twitter_possibility_query) if len(twitter_possibility_list): candidate_queryset = candidate_queryset.exclude(we_vote_id__in=twitter_possibility_list) except Exception as e: @@ -261,7 +262,8 @@ def fetch_number_of_candidates_needing_twitter_search(): one_month_ago = now() - timedelta(seconds=one_month_of_seconds) remote_request_query = remote_request_query.filter(datetime_of_action__gt=one_month_ago) remote_request_query = remote_request_query.filter(kind_of_action__iexact=RETRIEVE_POSSIBLE_TWITTER_HANDLES) - remote_request_list = remote_request_query.values_list('candidate_campaign_we_vote_id', flat=True).distinct() + remote_request_query = remote_request_query.values_list('candidate_campaign_we_vote_id', flat=True).distinct() + remote_request_list = list(remote_request_query) if len(remote_request_list): candidate_queryset = candidate_queryset.exclude(we_vote_id__in=remote_request_list) except Exception as e: @@ -416,7 +418,8 @@ def twitter_identity_retrieve_for_api(twitter_handle, voter_device_id=''): # tw google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, candidate_twitter_handle=twitter_handle, - candidate_name=candidate_name) + candidate_name=candidate_name, + read_only=True) if candidate_results['candidate_list_found']: candidate_list = candidate_results['candidate_list'] @@ -1310,10 +1313,11 @@ def retrieve_possible_twitter_handles_in_bulk( if positive_value_exists(state_code): candidate_queryset = candidate_queryset.filter(state_code__iexact=state_code) - # Exclude candidates we have already have TwitterLinkPossibility data for + # Exclude candidates we already have TwitterLinkPossibility data for try: - twitter_possibility_list = TwitterLinkPossibility.objects. \ + twitter_possibility_query = TwitterLinkPossibility.objects. \ values_list('candidate_campaign_we_vote_id', flat=True).distinct() + twitter_possibility_list = list(twitter_possibility_query) if len(twitter_possibility_list): candidate_queryset = candidate_queryset.exclude(we_vote_id__in=twitter_possibility_list) except Exception as e: @@ -1326,7 +1330,8 @@ def retrieve_possible_twitter_handles_in_bulk( one_month_ago = now() - timedelta(seconds=one_month_of_seconds) remote_request_query = remote_request_query.filter(datetime_of_action__gt=one_month_ago) remote_request_query = remote_request_query.filter(kind_of_action__iexact=RETRIEVE_POSSIBLE_TWITTER_HANDLES) - remote_request_list = remote_request_query.values_list('candidate_campaign_we_vote_id', flat=True).distinct() + remote_request_query = remote_request_query.values_list('candidate_campaign_we_vote_id', flat=True).distinct() + remote_request_list = list(remote_request_query) if len(remote_request_list): candidate_queryset = candidate_queryset.exclude(we_vote_id__in=remote_request_list) except Exception as e: @@ -1666,7 +1671,8 @@ def scrape_and_save_social_media_for_candidates_in_one_election(google_civic_ele results = candidate_list_manager.retrieve_all_candidates_for_upcoming_election( google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, - return_list_of_objects=return_list_of_objects) + return_list_of_objects=return_list_of_objects, + read_only=False) status += results['status'] if results['success']: candidate_list = results['candidate_list_objects'] @@ -1717,7 +1723,8 @@ def refresh_twitter_candidate_details_for_election(google_civic_election_id, sta candidates_results = candidate_list_manager.retrieve_all_candidates_for_upcoming_election( google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, - return_list_of_objects=return_list_of_objects) + return_list_of_objects=return_list_of_objects, + read_only=False) if candidates_results['candidate_list_found']: candidate_list = candidates_results['candidate_list_objects'] @@ -1767,7 +1774,8 @@ def transfer_candidate_twitter_handles_from_google_civic(google_civic_election_i results = candidate_list_object.retrieve_all_candidates_for_upcoming_election( google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, - return_list_of_objects=return_list_of_objects) + return_list_of_objects=return_list_of_objects, + read_only=False) status += results['status'] if results['success']: candidate_list = results['candidate_list_objects'] @@ -2087,7 +2095,7 @@ def twitter_sign_in_request_access_token_for_api(voter_device_id, return results voter_manager = VoterManager() - results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if not positive_value_exists(results['voter_found']): results = { 'status': "VALID_VOTER_MISSING", diff --git a/import_export_twitter/views_admin.py b/import_export_twitter/views_admin.py index a489457a5..e5f070d32 100644 --- a/import_export_twitter/views_admin.py +++ b/import_export_twitter/views_admin.py @@ -38,7 +38,7 @@ def delete_possible_twitter_handles_view(request, candidate_we_vote_id): return redirect_to_sign_in_page(request, authority_required) candidate_manager = CandidateManager() - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if not results['candidate_found']: messages.add_message(request, messages.INFO, results['status']) @@ -61,7 +61,7 @@ def retrieve_possible_twitter_handles_view(request, candidate_we_vote_id): return redirect_to_sign_in_page(request, authority_required) candidate_manager = CandidateManager() - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if not results['candidate_found']: messages.add_message(request, messages.INFO, results['status']) diff --git a/measure/controllers.py b/measure/controllers.py index 78f15c237..53b29ef6e 100644 --- a/measure/controllers.py +++ b/measure/controllers.py @@ -375,6 +375,7 @@ def measures_import_from_structured_json(structured_json): # Consumes measuresS 'measure_text': one_measure['measure_text'] if 'measure_text' in one_measure else '', 'measure_title': measure_title, 'measure_url': one_measure['measure_url'] if 'measure_url' in one_measure else '', + 'measure_year': one_measure['measure_year'] if 'measure_year' in one_measure else '', 'ocd_division_id': one_measure['ocd_division_id'] if 'ocd_division_id' in one_measure else '', 'primary_party': one_measure['primary_party'] if 'primary_party' in one_measure else '', 'state_code': state_code, diff --git a/measure/models.py b/measure/models.py index 30e9e345c..619198bc3 100644 --- a/measure/models.py +++ b/measure/models.py @@ -1062,7 +1062,7 @@ def fetch_measures_from_non_unique_identifiers_count( if keep_looking_for_duplicates and positive_value_exists(measure_title): # Search by ContestMeasure name exact match try: - contest_measure_query = ContestMeasure.objects.all() + contest_measure_query = ContestMeasure.objects.using('readonly').all() contest_measure_query = contest_measure_query.filter(measure_title__iexact=measure_title, google_civic_election_id=google_civic_election_id) if positive_value_exists(state_code): @@ -1081,8 +1081,14 @@ def fetch_measures_from_non_unique_identifiers_count( return 0 - def retrieve_measures(self, google_civic_election_id=0, ballotpedia_district_id=0, state_code="", limit=0, - read_only=False): + def retrieve_measures( + self, + google_civic_election_id=0, + ballotpedia_district_id=0, + state_code="", + limit=0, + measure_we_vote_id_list=[], + read_only=False): measure_list_objects = [] measure_list_light = [] measure_list_found = False @@ -1096,6 +1102,8 @@ def retrieve_measures(self, google_civic_election_id=0, ballotpedia_district_id= measure_queryset = measure_queryset.filter(google_civic_election_id=google_civic_election_id) if positive_value_exists(ballotpedia_district_id): measure_queryset = measure_queryset.filter(ballotpedia_district_id=ballotpedia_district_id) + if positive_value_exists(len(measure_we_vote_id_list)): + measure_queryset = measure_queryset.filter(we_vote_id__in=measure_we_vote_id_list) if positive_value_exists(state_code): measure_queryset = measure_queryset.filter(state_code__iexact=state_code) if positive_value_exists(limit): diff --git a/measure/views_admin.py b/measure/views_admin.py index f117bbc3a..95035eb04 100644 --- a/measure/views_admin.py +++ b/measure/views_admin.py @@ -186,7 +186,7 @@ def measures_sync_out_view(request): # measuresSyncOut 'google_civic_measure_title', 'google_civic_measure_title2', 'google_civic_measure_title3', 'google_civic_measure_title4', 'google_civic_measure_title5', 'maplight_id', - 'measure_subtitle', 'measure_text', 'measure_title', 'measure_url', + 'measure_subtitle', 'measure_text', 'measure_title', 'measure_url', 'measure_year', 'ocd_division_id', 'primary_party', 'state_code', 'vote_smart_id', @@ -566,20 +566,20 @@ def measure_new_view(request): google_civic_election_id = request.GET.get('google_civic_election_id', 0) - try: - measure_list = ContestMeasure.objects.order_by('measure_title') - if positive_value_exists(google_civic_election_id): - measure_list = measure_list.filter(google_civic_election_id=google_civic_election_id) - except ContestMeasure.DoesNotExist: - # This is fine - measure_list = ContestMeasure() - pass + # try: + # measure_list = ContestMeasure.objects.order_by('measure_title') + # if positive_value_exists(google_civic_election_id): + # measure_list = measure_list.filter(google_civic_election_id=google_civic_election_id) + # except ContestMeasure.DoesNotExist: + # # This is fine + # measure_list = ContestMeasure() + # pass messages_on_stage = get_messages(request) template_values = { 'messages_on_stage': messages_on_stage, 'google_civic_election_id': google_civic_election_id, - 'measure_list': measure_list, + # 'measure_list': measure_list, } return render(request, 'measure/measure_edit.html', template_values) @@ -664,17 +664,18 @@ def measure_edit_process_view(request): ballotpedia_measure_url = request.POST.get('ballotpedia_measure_url', False) ballotpedia_no_vote_description = request.POST.get('ballotpedia_no_vote_description', False) ballotpedia_yes_vote_description = request.POST.get('ballotpedia_yes_vote_description', False) - measure_id = convert_to_int(request.POST['measure_id']) - measure_title = request.POST.get('measure_title', False) google_civic_election_id = request.POST.get('google_civic_election_id', False) google_civic_measure_title = request.POST.get('google_civic_measure_title', False) google_civic_measure_title2 = request.POST.get('google_civic_measure_title2', False) google_civic_measure_title3 = request.POST.get('google_civic_measure_title3', False) google_civic_measure_title4 = request.POST.get('google_civic_measure_title4', False) google_civic_measure_title5 = request.POST.get('google_civic_measure_title5', False) + measure_id = convert_to_int(request.POST['measure_id']) + measure_title = request.POST.get('measure_title', False) measure_subtitle = request.POST.get('measure_subtitle', False) measure_text = request.POST.get('measure_text', False) measure_url = request.POST.get('measure_url', False) + measure_year = request.POST.get('measure_year', False) maplight_id = request.POST.get('maplight_id', False) vote_smart_id = request.POST.get('vote_smart_id', False) state_code = request.POST.get('state_code', False) @@ -698,9 +699,15 @@ def measure_edit_process_view(request): if measure_on_stage_found: # Update if ballotpedia_district_id is not False: - measure_on_stage.ballotpedia_district_id = ballotpedia_district_id + if ballotpedia_district_id == '': + measure_on_stage.ballotpedia_district_id = 0 + else: + measure_on_stage.ballotpedia_district_id = ballotpedia_district_id if ballotpedia_election_id is not False: - measure_on_stage.ballotpedia_election_id = ballotpedia_election_id + if ballotpedia_election_id == '': + measure_on_stage.ballotpedia_election_id = 0 + else: + measure_on_stage.ballotpedia_election_id = ballotpedia_election_id if ballotpedia_measure_status is not False: measure_on_stage.ballotpedia_measure_status = ballotpedia_measure_status if ballotpedia_measure_url is not False: @@ -729,6 +736,8 @@ def measure_edit_process_view(request): measure_on_stage.measure_text = measure_text if measure_url is not False: measure_on_stage.measure_url = measure_url + if measure_year is not False: + measure_on_stage.measure_year = measure_year if maplight_id is not False: measure_on_stage.maplight_id = maplight_id if vote_smart_id is not False: @@ -745,8 +754,6 @@ def measure_edit_process_view(request): else: # Create new measure_on_stage = ContestMeasure( - ballotpedia_district_id=ballotpedia_district_id, - ballotpedia_election_id=ballotpedia_election_id, ballotpedia_measure_status=ballotpedia_measure_status, ballotpedia_measure_url=ballotpedia_measure_url, ballotpedia_no_vote_description=ballotpedia_no_vote_description, @@ -761,14 +768,25 @@ def measure_edit_process_view(request): measure_text=measure_text, measure_title=measure_title, measure_url=measure_url, + measure_year=measure_year, state_code=state_code, maplight_id=maplight_id, vote_smart_id=vote_smart_id, ) + if ballotpedia_district_id is not False: + if ballotpedia_district_id == '': + measure_on_stage.ballotpedia_district_id = 0 + else: + measure_on_stage.ballotpedia_district_id = ballotpedia_district_id + if ballotpedia_election_id is not False: + if ballotpedia_election_id == '': + measure_on_stage.ballotpedia_election_id = 0 + else: + measure_on_stage.ballotpedia_election_id = ballotpedia_election_id measure_on_stage.save() messages.add_message(request, messages.INFO, 'New measure saved.') except Exception as e: - messages.add_message(request, messages.ERROR, 'Could not save measure.') + messages.add_message(request, messages.ERROR, 'Could not save measure: ' + str(e)) return HttpResponseRedirect(reverse('measure:measure_list', args=()) + "?google_civic_election_id=" + str(google_civic_election_id) + diff --git a/office/models.py b/office/models.py index 43f59c2e0..568534cb4 100644 --- a/office/models.py +++ b/office/models.py @@ -234,6 +234,20 @@ class ContestOffice(models.Model): elected_office_name = models.CharField(verbose_name="name of the elected office", max_length=255, null=True, blank=True, default=None) + def get_election_day_text(self): + if positive_value_exists(self.google_civic_election_id): + try: + from election.models import Election + one_election = Election.objects.using('readonly')\ + .get(google_civic_election_id=self.google_civic_election_id) + return one_election.election_day_text + except Exception as e: + handle_record_found_more_than_one_exception(e, logger=logger) + logger.error("office.get_election_day_text:" + str(e)) + return "" + else: + return "" + def get_office_state(self): if positive_value_exists(self.state_code): return self.state_code @@ -909,7 +923,8 @@ def fetch_contest_office_id_from_we_vote_id(self, contest_office_we_vote_id): contest_office_id = 0 try: if positive_value_exists(contest_office_we_vote_id): - contest_office_on_stage = ContestOffice.objects.get(we_vote_id=contest_office_we_vote_id) + contest_office_on_stage = ContestOffice.objects.using('readonly').get( + we_vote_id=contest_office_we_vote_id) contest_office_id = contest_office_on_stage.id except ContestOffice.MultipleObjectsReturned as e: @@ -929,7 +944,8 @@ def fetch_google_civic_election_id_from_office_we_vote_id(self, contest_office_w google_civic_election_id = '0' try: if positive_value_exists(contest_office_we_vote_id): - contest_office_on_stage = ContestOffice.objects.get(we_vote_id=contest_office_we_vote_id) + contest_office_on_stage = ContestOffice.objects.using('readonly').get( + we_vote_id=contest_office_we_vote_id) google_civic_election_id = contest_office_on_stage.google_civic_election_id except ContestOffice.MultipleObjectsReturned as e: @@ -949,7 +965,8 @@ def fetch_state_code_from_we_vote_id(self, contest_office_we_vote_id): state_code = "" try: if positive_value_exists(contest_office_we_vote_id): - contest_office_on_stage = ContestOffice.objects.get(we_vote_id=contest_office_we_vote_id) + contest_office_on_stage = ContestOffice.objects.using('readonly').get( + we_vote_id=contest_office_we_vote_id) state_code = contest_office_on_stage.state_code except ContestOffice.MultipleObjectsReturned as e: @@ -1172,7 +1189,7 @@ def count_contest_offices_for_election(self, google_civic_election_id): success = False if positive_value_exists(google_civic_election_id): try: - contest_office_item_queryset = ContestOffice.objects.all() + contest_office_item_queryset = ContestOffice.objects.using('readonly').all() contest_office_item_queryset = contest_office_item_queryset.filter( google_civic_election_id=google_civic_election_id) contest_offices_count = contest_office_item_queryset.count() @@ -1249,7 +1266,7 @@ def fetch_offices_from_non_unique_identifiers_count( if keep_looking_for_duplicates and positive_value_exists(office_name): # Search by Contest Office name exact match try: - contest_office_query = ContestOffice.objects.all() + contest_office_query = ContestOffice.objects.using('readonly').all() contest_office_query = contest_office_query.filter(office_name__iexact=office_name, google_civic_election_id=google_civic_election_id) if positive_value_exists(state_code): diff --git a/organization/controllers.py b/organization/controllers.py index c77375608..799fbd54b 100644 --- a/organization/controllers.py +++ b/organization/controllers.py @@ -10,6 +10,7 @@ import robot_detection import tweepy from PIL import Image, ImageOps +from django.db.models import Q from django.http import HttpResponse import wevote_functions.admin @@ -233,7 +234,10 @@ def full_domain_string_available(full_domain_string, requesting_organization_id) try: organization_list_query = Organization.objects.using('readonly').all() organization_list_query = organization_list_query.exclude(id=requesting_organization_id) - organization_list_query = organization_list_query.filter(chosen_domain_string__iexact=full_domain_string) + organization_list_query = organization_list_query.filter( + Q(chosen_domain_string__iexact=full_domain_string) | + Q(chosen_domain_string2__iexact=full_domain_string) | + Q(chosen_domain_string3__iexact=full_domain_string)) organization_domain_match_count = organization_list_query.count() if positive_value_exists(organization_domain_match_count): status += "FULL_DOMAIN_STRING_FOUND-OWNED_BY_ORGANIZATION " @@ -2176,6 +2180,8 @@ def organization_retrieve_for_api( # organizationRetrieve 'status': status, 'success': False, 'chosen_domain_string': '', + 'chosen_domain_string2': '', + 'chosen_domain_string3': '', 'chosen_favicon_url_https': '', 'chosen_feature_package': '', 'chosen_google_analytics_tracking_id': '', @@ -2253,6 +2259,8 @@ def organization_retrieve_for_api( # organizationRetrieve 'success': True, 'status': status, 'chosen_domain_string': organization.chosen_domain_string, + 'chosen_domain_string2': organization.chosen_domain_string2, + 'chosen_domain_string3': organization.chosen_domain_string3, 'chosen_favicon_url_https': organization.chosen_favicon_url_https, 'chosen_feature_package': organization.chosen_feature_package, 'chosen_google_analytics_tracking_id': organization.chosen_google_analytics_tracking_id, @@ -2311,6 +2319,8 @@ def organization_retrieve_for_api( # organizationRetrieve 'status': status, 'success': False, 'chosen_domain_string': '', + 'chosen_domain_string2': '', + 'chosen_domain_string3': '', 'chosen_favicon_url_https': '', 'chosen_google_analytics_tracking_id': '', 'chosen_html_verification_string': '', @@ -2368,6 +2378,8 @@ def organization_save_for_api( # organizationSave facebook_email=False, facebook_profile_image_url_https=False, chosen_domain_string=False, + chosen_domain_string2=False, + chosen_domain_string3=False, chosen_google_analytics_tracking_id=False, chosen_html_verification_string=False, chosen_hide_we_vote_logo=None, @@ -2397,6 +2409,8 @@ def organization_save_for_api( # organizationSave :param facebook_email: :param facebook_profile_image_url_https: :param chosen_domain_string: + :param chosen_domain_string2: + :param chosen_domain_string3: :param chosen_google_analytics_tracking_id: :param chosen_html_verification_string: :param chosen_hide_we_vote_logo: @@ -2430,6 +2444,8 @@ def organization_save_for_api( # organizationSave 'status': "ORGANIZATION_REQUIRED_UNIQUE_IDENTIFIER_VARIABLES_MISSING", 'success': False, 'chosen_domain_string': chosen_domain_string, + 'chosen_domain_string2': chosen_domain_string2, + 'chosen_domain_string3': chosen_domain_string3, 'chosen_favicon_url_https': '', 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, 'chosen_html_verification_string': chosen_html_verification_string, @@ -2468,6 +2484,8 @@ def organization_save_for_api( # organizationSave 'status': "NEW_ORGANIZATION_REQUIRED_VARIABLES_MISSING", 'success': False, 'chosen_domain_string': chosen_domain_string, + 'chosen_domain_string2': chosen_domain_string2, + 'chosen_domain_string3': chosen_domain_string3, 'chosen_favicon_url_https': '', 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, 'chosen_html_verification_string': chosen_html_verification_string, @@ -2570,6 +2588,8 @@ def organization_save_for_api( # organizationSave facebook_profile_image_url_https=facebook_profile_image_url_https, facebook_background_image_url_https=facebook_background_image_url_https, chosen_domain_string=chosen_domain_string, + chosen_domain_string2=chosen_domain_string2, + chosen_domain_string3=chosen_domain_string3, chosen_google_analytics_tracking_id=chosen_google_analytics_tracking_id, chosen_html_verification_string=chosen_html_verification_string, chosen_hide_we_vote_logo=chosen_hide_we_vote_logo, @@ -2694,10 +2714,12 @@ def organization_save_for_api( # organizationSave organization_banner_url = organization_banner_url[0] results = { - 'success': success, - 'status': status, - 'voter_device_id': voter_device_id, + 'success': success, + 'status': status, + 'voter_device_id': voter_device_id, 'chosen_domain_string': organization.chosen_domain_string, + 'chosen_domain_string2': organization.chosen_domain_string2, + 'chosen_domain_string3': organization.chosen_domain_string3, 'chosen_favicon_url_https': organization.chosen_favicon_url_https, 'chosen_google_analytics_tracking_id': organization.chosen_google_analytics_tracking_id, 'chosen_html_verification_string': organization.chosen_html_verification_string, @@ -2712,11 +2734,11 @@ def organization_save_for_api( # organizationSave 'chosen_subscription_plan': organization.chosen_subscription_plan, 'subscription_plan_end_day_text': organization.subscription_plan_end_day_text, 'subscription_plan_features_active': organization.subscription_plan_features_active, - 'chosen_feature_package': organization.chosen_feature_package, - 'features_provided_bitmap': organization.features_provided_bitmap, - 'organization_id': organization.id, - 'organization_we_vote_id': organization.we_vote_id, - 'new_organization_created': save_results['new_organization_created'], + 'chosen_feature_package': organization.chosen_feature_package, + 'features_provided_bitmap': organization.features_provided_bitmap, + 'organization_id': organization.id, + 'organization_we_vote_id': organization.we_vote_id, + 'new_organization_created': save_results['new_organization_created'], 'organization_name': organization.organization_name if positive_value_exists(organization.organization_name) else '', 'organization_description': @@ -2731,60 +2753,63 @@ def organization_save_for_api( # organizationSave 'organization_instagram_handle': organization.organization_instagram_handle if positive_value_exists(organization.organization_instagram_handle) else '', - 'organization_banner_url': organization_banner_url, - 'organization_photo_url': organization.organization_photo_url() + 'organization_banner_url': organization_banner_url, + 'organization_photo_url': organization.organization_photo_url() if positive_value_exists(organization.organization_photo_url()) else '', - 'organization_photo_url_large': we_vote_hosted_profile_image_url_large, - 'organization_photo_url_medium': organization.we_vote_hosted_profile_image_url_medium, - 'organization_photo_url_tiny': organization.we_vote_hosted_profile_image_url_tiny, - 'organization_twitter_handle': organization.organization_twitter_handle if positive_value_exists( + 'organization_photo_url_large': we_vote_hosted_profile_image_url_large, + 'organization_photo_url_medium': organization.we_vote_hosted_profile_image_url_medium, + 'organization_photo_url_tiny': organization.we_vote_hosted_profile_image_url_tiny, + 'organization_twitter_handle': organization.organization_twitter_handle if positive_value_exists( organization.organization_twitter_handle) else '', - 'organization_type': organization.organization_type if positive_value_exists( + 'organization_type': organization.organization_type if positive_value_exists( organization.organization_type) else '', - 'twitter_followers_count': organization.twitter_followers_count if positive_value_exists( + 'twitter_followers_count': organization.twitter_followers_count if positive_value_exists( organization.twitter_followers_count) else 0, - 'twitter_description': organization.twitter_description if positive_value_exists( + 'twitter_description': organization.twitter_description if positive_value_exists( organization.twitter_description) else '', - 'refresh_from_twitter': refresh_from_twitter, - 'facebook_id': organization.facebook_id if positive_value_exists(organization.facebook_id) else 0, + 'refresh_from_twitter': refresh_from_twitter, + 'facebook_id': organization.facebook_id if positive_value_exists( + organization.facebook_id) else 0, } return results else: results = { - 'success': False, - 'status': save_results['status'], - 'voter_device_id': voter_device_id, - 'chosen_domain_string': chosen_domain_string, - 'chosen_favicon_url_https': '', - 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, - 'chosen_html_verification_string': chosen_html_verification_string, - 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, - 'chosen_logo_url_https': '', - 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, - 'chosen_ready_introduction_text': chosen_ready_introduction_text, - 'chosen_ready_introduction_title': chosen_ready_introduction_title, - 'chosen_social_share_description': chosen_social_share_description, + 'success': False, + 'status': save_results['status'], + 'voter_device_id': voter_device_id, + 'chosen_domain_string': chosen_domain_string, + 'chosen_domain_string2': chosen_domain_string2, + 'chosen_domain_string3': chosen_domain_string3, + 'chosen_favicon_url_https': '', + 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, + 'chosen_html_verification_string': chosen_html_verification_string, + 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, + 'chosen_logo_url_https': '', + 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, + 'chosen_ready_introduction_text': chosen_ready_introduction_text, + 'chosen_ready_introduction_title': chosen_ready_introduction_title, + 'chosen_social_share_description': chosen_social_share_description, 'chosen_social_share_image_256x256_url_https': '', - 'chosen_subdomain_string': chosen_subdomain_string, - 'chosen_subscription_plan': chosen_subscription_plan, - 'subscription_plan_end_day_text': '', - 'subscription_plan_features_active': '', - 'chosen_feature_package': '', - 'features_provided_bitmap': '', - 'organization_id': organization_id, - 'organization_we_vote_id': organization_we_vote_id, - 'new_organization_created': save_results['new_organization_created'], - 'organization_name': organization_name, - 'organization_email': organization_email, - 'organization_website': organization_website, - 'organization_facebook': organization_facebook, - 'organization_photo_url': organization_image, - 'organization_twitter_handle': organization_twitter_handle, - 'organization_type': organization_type, - 'twitter_followers_count': 0, - 'twitter_description': "", - 'refresh_from_twitter': refresh_from_twitter, - 'facebook_id': facebook_id, + 'chosen_subdomain_string': chosen_subdomain_string, + 'chosen_subscription_plan': chosen_subscription_plan, + 'subscription_plan_end_day_text': '', + 'subscription_plan_features_active': '', + 'chosen_feature_package': '', + 'features_provided_bitmap': '', + 'organization_id': organization_id, + 'organization_we_vote_id': organization_we_vote_id, + 'new_organization_created': save_results['new_organization_created'], + 'organization_name': organization_name, + 'organization_email': organization_email, + 'organization_website': organization_website, + 'organization_facebook': organization_facebook, + 'organization_photo_url': organization_image, + 'organization_twitter_handle': organization_twitter_handle, + 'organization_type': organization_type, + 'twitter_followers_count': 0, + 'twitter_description': "", + 'refresh_from_twitter': refresh_from_twitter, + 'facebook_id': facebook_id, } return results @@ -3091,20 +3116,20 @@ def site_configuration_retrieve_for_api(hostname): # siteConfigurationRetrieve if not positive_value_exists(hostname): status += "HOSTNAME_MISSING " results = { - 'success': success, - 'status': status, - 'chosen_about_organization_external_url': chosen_about_organization_external_url, - 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, - 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, - 'chosen_logo_url_https': chosen_logo_url_https, - 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, - 'chosen_ready_introduction_text': chosen_ready_introduction_text, - 'chosen_ready_introduction_title': chosen_ready_introduction_title, - 'chosen_website_name': chosen_website_name, - 'features_provided_bitmap': features_provided_bitmap, - 'hostname': hostname, - 'organization_we_vote_id': organization_we_vote_id, - 'reserved_by_we_vote': reserved_by_we_vote, + 'success': success, + 'status': status, + 'chosen_about_organization_external_url': chosen_about_organization_external_url, + 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, + 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, + 'chosen_logo_url_https': chosen_logo_url_https, + 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, + 'chosen_ready_introduction_text': chosen_ready_introduction_text, + 'chosen_ready_introduction_title': chosen_ready_introduction_title, + 'chosen_website_name': chosen_website_name, + 'features_provided_bitmap': features_provided_bitmap, + 'hostname': hostname, + 'organization_we_vote_id': organization_we_vote_id, + 'reserved_by_we_vote': reserved_by_we_vote, } return results @@ -3118,20 +3143,20 @@ def site_configuration_retrieve_for_api(hostname): # siteConfigurationRetrieve success = False hostname = "" results = { - 'success': success, - 'status': status, - 'chosen_about_organization_external_url': chosen_about_organization_external_url, - 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, - 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, - 'chosen_logo_url_https': chosen_logo_url_https, - 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, - 'chosen_ready_introduction_text': chosen_ready_introduction_text, - 'chosen_ready_introduction_title': chosen_ready_introduction_title, - 'chosen_website_name': chosen_website_name, - 'features_provided_bitmap': features_provided_bitmap, - 'hostname': hostname, - 'organization_we_vote_id': organization_we_vote_id, - 'reserved_by_we_vote': reserved_by_we_vote, + 'success': success, + 'status': status, + 'chosen_about_organization_external_url': chosen_about_organization_external_url, + 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, + 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, + 'chosen_logo_url_https': chosen_logo_url_https, + 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, + 'chosen_ready_introduction_text': chosen_ready_introduction_text, + 'chosen_ready_introduction_title': chosen_ready_introduction_title, + 'chosen_website_name': chosen_website_name, + 'features_provided_bitmap': features_provided_bitmap, + 'hostname': hostname, + 'organization_we_vote_id': organization_we_vote_id, + 'reserved_by_we_vote': reserved_by_we_vote, } return results results = organization_manager.retrieve_organization_from_incoming_hostname(hostname, read_only=True) @@ -3163,20 +3188,20 @@ def site_configuration_retrieve_for_api(hostname): # siteConfigurationRetrieve status += "HOSTNAME_NOT_OWNED_BY_ORG_OR_RESERVED_BY_WE_VOTE " results = { - 'success': success, - 'status': status, - 'chosen_about_organization_external_url': chosen_about_organization_external_url, - 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, - 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, - 'chosen_logo_url_https': chosen_logo_url_https, - 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, - 'chosen_ready_introduction_text': chosen_ready_introduction_text, - 'chosen_ready_introduction_title': chosen_ready_introduction_title, - 'chosen_website_name': chosen_website_name, - 'features_provided_bitmap': features_provided_bitmap, - 'hostname': hostname, - 'organization_we_vote_id': organization_we_vote_id, - 'reserved_by_we_vote': reserved_by_we_vote, + 'success': success, + 'status': status, + 'chosen_about_organization_external_url': chosen_about_organization_external_url, + 'chosen_google_analytics_tracking_id': chosen_google_analytics_tracking_id, + 'chosen_hide_we_vote_logo': chosen_hide_we_vote_logo, + 'chosen_logo_url_https': chosen_logo_url_https, + 'chosen_prevent_sharing_opinions': chosen_prevent_sharing_opinions, + 'chosen_ready_introduction_text': chosen_ready_introduction_text, + 'chosen_ready_introduction_title': chosen_ready_introduction_title, + 'chosen_website_name': chosen_website_name, + 'features_provided_bitmap': features_provided_bitmap, + 'hostname': hostname, + 'organization_we_vote_id': organization_we_vote_id, + 'reserved_by_we_vote': reserved_by_we_vote, } return results diff --git a/organization/models.py b/organization/models.py index cc8ae8745..9bad9ba31 100644 --- a/organization/models.py +++ b/organization/models.py @@ -70,6 +70,8 @@ 'ballotpedia_page_title', 'ballotpedia_photo_url', 'chosen_domain_string', + 'chosen_domain_string2', + 'chosen_domain_string3', 'chosen_favicon_url_https', 'chosen_feature_package', 'chosen_google_analytics_tracking_id', @@ -550,7 +552,7 @@ def retrieve_organization_from_twitter_user_id_old(self, twitter_user_id): def retrieve_organization_from_facebook_id(self, facebook_id): status = "" facebook_manager = FacebookManager() - results = facebook_manager.retrieve_facebook_link_to_voter_from_facebook_id(facebook_id) + results = facebook_manager.retrieve_facebook_link_to_voter_from_facebook_id(facebook_id, read_only=True) if results['facebook_link_to_voter_found']: facebook_link_to_voter = results['facebook_link_to_voter'] if positive_value_exists(facebook_link_to_voter.voter_we_vote_id): @@ -583,8 +585,15 @@ def retrieve_organization_from_facebook_id(self, facebook_id): } return results - def retrieve_organization(self, organization_id=None, we_vote_id=None, vote_smart_id=None, twitter_user_id=None, - incoming_hostname=None, organization_api_pass_code=False, read_only=False): + def retrieve_organization( + self, + organization_id=None, + we_vote_id=None, + vote_smart_id=None, + twitter_user_id=None, + incoming_hostname=None, + organization_api_pass_code=False, + read_only=False): """ Get an organization, based the passed in parameters :param organization_id: @@ -658,10 +667,14 @@ def retrieve_organization(self, organization_id=None, we_vote_id=None, vote_smar if read_only: organization_on_stage = Organization.objects.using('readonly')\ .get(Q(chosen_domain_string__iexact=incoming_hostname) | + Q(chosen_domain_string2__iexact=incoming_hostname) | + Q(chosen_domain_string3__iexact=incoming_hostname) | Q(chosen_subdomain_string__iexact=incoming_subdomain)) else: organization_on_stage = Organization.objects\ .get(Q(chosen_domain_string__iexact=incoming_hostname) | + Q(chosen_domain_string2__iexact=incoming_hostname) | + Q(chosen_domain_string3__iexact=incoming_hostname) | Q(chosen_subdomain_string__iexact=incoming_subdomain)) organization_on_stage_id = organization_on_stage.id status = "ORGANIZATION_FOUND_WITH_INCOMING_HOSTNAME " @@ -789,7 +802,7 @@ def retrieve_team_member_list( def fetch_external_voter_id(self, organization_we_vote_id, voter_we_vote_id): if positive_value_exists(organization_we_vote_id) and positive_value_exists(voter_we_vote_id): - link_query = OrganizationMembershipLinkToVoter.objects.all() + link_query = OrganizationMembershipLinkToVoter.objects.using('readonly').all() link_query = link_query.filter(organization_we_vote_id=organization_we_vote_id) link_query = link_query.filter(voter_we_vote_id=voter_we_vote_id) external_voter = link_query.first() @@ -801,7 +814,7 @@ def fetch_organization_id(self, we_vote_id): organization_id = 0 if positive_value_exists(we_vote_id): organization_manager = OrganizationManager() - results = organization_manager.retrieve_organization(organization_id, we_vote_id) + results = organization_manager.retrieve_organization(organization_id, we_vote_id, read_only=True) if results['success']: return results['organization_id'] return 0 @@ -1092,19 +1105,25 @@ def repair_organization(self, organization): def update_or_create_organization( self, organization_id, we_vote_id, - organization_website_search, organization_twitter_search, - organization_name=False, organization_description=False, + organization_website_search, + organization_twitter_search, + organization_name=False, + organization_description=False, organization_website=False, organization_twitter_handle=False, organization_email=False, organization_facebook=False, organization_instagram_handle=False, organization_image=False, - organization_type=False, refresh_from_twitter=False, - facebook_id=False, facebook_email=False, + organization_type=False, + refresh_from_twitter=False, + facebook_id=False, + facebook_email=False, facebook_profile_image_url_https=False, facebook_background_image_url_https=False, chosen_domain_string=False, + chosen_domain_string2=False, + chosen_domain_string3=False, chosen_google_analytics_tracking_id=False, chosen_html_verification_string=False, chosen_hide_we_vote_logo=None, @@ -1136,6 +1155,8 @@ def update_or_create_organization( :param facebook_profile_image_url_https: :param facebook_background_image_url_https: :param chosen_domain_string: + :param chosen_domain_string2: + :param chosen_domain_string3: :param chosen_google_analytics_tracking_id: :param chosen_html_verification_string: :param chosen_hide_we_vote_logo: @@ -1174,6 +1195,8 @@ def update_or_create_organization( organization_image = organization_image.strip() if organization_image is not False else False organization_type = organization_type.strip() if organization_type is not False else False chosen_domain_string = chosen_domain_string.strip() if chosen_domain_string is not False else False + chosen_domain_string2 = chosen_domain_string2.strip() if chosen_domain_string2 is not False else False + chosen_domain_string3 = chosen_domain_string3.strip() if chosen_domain_string3 is not False else False chosen_google_analytics_tracking_id = chosen_google_analytics_tracking_id.strip() \ if chosen_google_analytics_tracking_id is not False else False chosen_html_verification_string = chosen_html_verification_string.strip() \ @@ -1278,6 +1301,12 @@ def update_or_create_organization( if chosen_domain_string is not False: value_changed = True organization_on_stage.chosen_domain_string = chosen_domain_string + if chosen_domain_string2 is not False: + value_changed = True + organization_on_stage.chosen_domain_string2 = chosen_domain_string2 + if chosen_domain_string3 is not False: + value_changed = True + organization_on_stage.chosen_domain_string3 = chosen_domain_string3 if chosen_google_analytics_tracking_id is not False: value_changed = True organization_on_stage.chosen_google_analytics_tracking_id = \ @@ -1366,7 +1395,7 @@ def update_or_create_organization( # 3a) FacebookLinkToVoter exists? If not, go to step 3b if not organization_on_stage_found and positive_value_exists(facebook_id): facebook_manager = FacebookManager() - facebook_results = facebook_manager.retrieve_facebook_link_to_voter(facebook_id) + facebook_results = facebook_manager.retrieve_facebook_link_to_voter(facebook_id, read_only=True) if facebook_results['facebook_link_to_voter_found']: facebook_link_to_voter = facebook_results['facebook_link_to_voter'] voter_manager = VoterManager() @@ -1502,6 +1531,12 @@ def update_or_create_organization( if chosen_domain_string is not False: value_changed = True organization_on_stage.chosen_domain_string = chosen_domain_string + if chosen_domain_string2 is not False: + value_changed = True + organization_on_stage.chosen_domain_string2 = chosen_domain_string2 + if chosen_domain_string3 is not False: + value_changed = True + organization_on_stage.chosen_domain_string3 = chosen_domain_string3 if chosen_google_analytics_tracking_id is not False: value_changed = True organization_on_stage.chosen_google_analytics_tracking_id = \ @@ -1666,6 +1701,12 @@ def update_or_create_organization( if chosen_domain_string is not False: value_changed = True organization_on_stage.chosen_domain_string = chosen_domain_string + if chosen_domain_string2 is not False: + value_changed = True + organization_on_stage.chosen_domain_string2 = chosen_domain_string2 + if chosen_domain_string3 is not False: + value_changed = True + organization_on_stage.chosen_domain_string3 = chosen_domain_string3 if chosen_google_analytics_tracking_id is not False: value_changed = True organization_on_stage.chosen_google_analytics_tracking_id = \ @@ -2941,6 +2982,8 @@ class Organization(models.Model): # This is the domain name the client has configured for their We Vote configured site chosen_domain_string = models.CharField( verbose_name="client domain name for we vote site", max_length=255, null=True, blank=True) + chosen_domain_string2 = models.CharField(max_length=255, null=True, blank=True) # Alternate ex/ www + chosen_domain_string3 = models.CharField(max_length=255, null=True, blank=True) # Another alternate chosen_favicon_url_https = models.TextField( verbose_name='url of client favicon', blank=True, null=True) chosen_google_analytics_tracking_id = models.CharField(max_length=255, null=True, blank=True) diff --git a/organization/views_admin.py b/organization/views_admin.py index 0f65232f6..19c05f4b3 100644 --- a/organization/views_admin.py +++ b/organization/views_admin.py @@ -461,7 +461,11 @@ def organization_list_view(request): else: organization_list_query = organization_list_query.order_by('organization_name') - if positive_value_exists(show_organizations_without_email): + # wv02org35759 + if positive_value_exists(organization_search): + # Do not limit search + pass + elif positive_value_exists(show_organizations_without_email): organization_list_query = organization_list_query.filter( Q(organization_email__isnull=True) | Q(organization_email__exact='') @@ -486,6 +490,9 @@ def organization_list_view(request): ) else: organization_list_query = organization_list_query.filter(organization_type__iexact=organization_type_filter) + elif positive_value_exists(organization_search): + # Do not remove individuals from search + pass else: # By default, don't show individuals organization_list_query = organization_list_query.exclude(organization_type__iexact=INDIVIDUAL) @@ -524,6 +531,12 @@ def organization_list_view(request): new_filter = Q(chosen_domain_string__icontains=one_word) filters.append(new_filter) + new_filter = Q(chosen_domain_string2__icontains=one_word) + filters.append(new_filter) + + new_filter = Q(chosen_domain_string3__icontains=one_word) + filters.append(new_filter) + new_filter = Q(chosen_subdomain_string__icontains=one_word) filters.append(new_filter) @@ -1584,6 +1597,8 @@ def organization_edit_account_process_view(request): organization_id = convert_to_int(request.POST.get('organization_id', 0)) chosen_about_organization_external_url = request.POST.get('chosen_about_organization_external_url', None) chosen_domain_string = request.POST.get('chosen_domain_string', None) + chosen_domain_string2 = request.POST.get('chosen_domain_string2', None) + chosen_domain_string3 = request.POST.get('chosen_domain_string3', None) chosen_domain_type_is_campaign = request.POST.get('chosen_domain_type_is_campaign', None) chosen_favicon_url_https = request.POST.get('chosen_favicon_url_https', None) chosen_google_analytics_tracking_id = request.POST.get('chosen_google_analytics_tracking_id', None) @@ -1637,6 +1652,32 @@ def organization_edit_account_process_view(request): status += domain_results['status'] else: organization_on_stage.chosen_domain_string = None + if chosen_domain_string2 is not None: + if positive_value_exists(chosen_domain_string2): + domain_results = full_domain_string_available(chosen_domain_string2, + requesting_organization_id=organization_id) + if domain_results['full_domain_string_available']: + organization_on_stage.chosen_domain_string2 = chosen_domain_string2.strip() + else: + message = 'Cannot save chosen_domain_string2: \'' + chosen_domain_string2 + '\', status: ' + \ + domain_results['status'] + messages.add_message(request, messages.ERROR, message) + status += domain_results['status'] + else: + organization_on_stage.chosen_domain_string2 = None + if chosen_domain_string3 is not None: + if positive_value_exists(chosen_domain_string3): + domain_results = full_domain_string_available(chosen_domain_string3, + requesting_organization_id=organization_id) + if domain_results['full_domain_string_available']: + organization_on_stage.chosen_domain_string3 = chosen_domain_string3.strip() + else: + message = 'Cannot save chosen_domain_string3: \'' + chosen_domain_string3 + '\', status: ' + \ + domain_results['status'] + messages.add_message(request, messages.ERROR, message) + status += domain_results['status'] + else: + organization_on_stage.chosen_domain_string3 = None if chosen_domain_type_is_campaign is not None: organization_on_stage.chosen_domain_type_is_campaign = chosen_domain_type_is_campaign if chosen_favicon_url_https is not None: @@ -1725,7 +1766,7 @@ def organization_edit_account_process_view(request): else: status += "CHOSEN_SUBDOMAIN_ALREADY_EXISTS " - # add domain to aws route53 DNS + # add subdomain to aws route53 DNS route53_results = add_subdomain_route53_record(chosen_subdomain_string) if route53_results['success']: status += "SUBDOMAIN_ROUTE53_ADDED " @@ -1926,7 +1967,7 @@ def organization_position_list_view(request, organization_id=0, organization_we_ candidate_manager = CandidateManager() if positive_value_exists(candidate_we_vote_id): candidate_id = 0 - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] candidate_id = candidate.id @@ -2199,7 +2240,7 @@ def organization_position_new_view(request, organization_id): candidate_manager = CandidateManager() if positive_value_exists(candidate_we_vote_id): candidate_id = 0 - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] candidate_id = candidate.id @@ -2248,7 +2289,8 @@ def organization_position_new_view(request, organization_id): google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, search_string=candidate_search, - return_list_of_objects=True) + return_list_of_objects=True, + read_only=True) if results['candidate_list_found']: candidates_for_this_election_list = results['candidate_list_objects'] @@ -2946,8 +2988,11 @@ def reserved_domain_edit_process_view(request): try: organization_list_query = Organization.objects.using('readonly').all() if positive_value_exists(full_domain_string): - organization_list_query = organization_list_query.\ - filter(chosen_domain_string__iexact=full_domain_string) + organization_list_query = organization_list_query.filter( + Q(chosen_domain_string__iexact=full_domain_string) | + Q(chosen_domain_string2__iexact=full_domain_string) | + Q(chosen_domain_string3__iexact=full_domain_string)) + else: organization_list_query = organization_list_query.\ filter(chosen_subdomain_string__iexact=subdomain_string) @@ -3112,7 +3157,10 @@ def reserved_domain_list_view(request): # exclude(chosen_subdomain_string__isnull=True). \ # exclude(chosen_subdomain_string__exact='') if positive_value_exists(show_full_domains) and not positive_value_exists(show_subdomains): - organization_domain_list_query = organization_domain_list_query.filter(chosen_domain_string__isnull=False) + organization_domain_list_query = organization_domain_list_query.filter( + Q(chosen_domain_string__isnull=False) | + Q(chosen_domain_string2__isnull=False) | + Q(chosen_domain_string3__isnull=False)) organization_domain_list_query = organization_domain_list_query.order_by('chosen_domain_string') elif positive_value_exists(show_subdomains) and not positive_value_exists(show_full_domains): organization_domain_list_query = organization_domain_list_query.filter(chosen_subdomain_string__isnull=False) @@ -3120,6 +3168,8 @@ def reserved_domain_list_view(request): else: organization_domain_list_query = organization_domain_list_query.filter( Q(chosen_domain_string__isnull=False) | + Q(chosen_domain_string2__isnull=False) | + Q(chosen_domain_string3__isnull=False) | Q(chosen_subdomain_string__isnull=False) ) organization_domain_list_query = organization_domain_list_query.order_by('chosen_subdomain_string').\ @@ -3132,6 +3182,14 @@ def reserved_domain_list_view(request): new_filter = Q(chosen_domain_string__icontains=one_word) filters.append(new_filter) + filters = [] + new_filter = Q(chosen_domain_string2__icontains=one_word) + filters.append(new_filter) + + filters = [] + new_filter = Q(chosen_domain_string3__icontains=one_word) + filters.append(new_filter) + new_filter = Q(chosen_subdomain_string__icontains=one_word) filters.append(new_filter) diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 000000000..2f44bd8eb --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1171 @@ +{ + "name": "WeVoteServer", + "version": "1.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "WeVoteServer", + "version": "1.0.0", + "license": "MIT", + "devDependencies": { + "react": "^0.14.3", + "react-redux": "^4.0.0", + "redux": "^3.0.4", + "redux-devtools": "^2.1.5", + "snyk": "^1.251.0" + } + }, + "node_modules/acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", + "dev": true, + "engines": { + "node": ">=0.4.2" + } + }, + "node_modules/asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "dev": true + }, + "node_modules/ast-types": { + "version": "0.9.6", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.9.6.tgz", + "integrity": "sha512-qEdtR2UH78yyHX/AUNfXmJTlM48XoFZKBdwi1nzkI1mJL21cmbu0cvjxjpkXJ5NENMq42H+hNs8VLJcqXLerBQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/babel-runtime": { + "version": "5.8.38", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-5.8.38.tgz", + "integrity": "sha512-KpgoA8VE/pMmNCrnEeeXqFG24TIH11Z3ZaimIhJWsin8EbfZy3WzFKUTIan10ZIDgRVvi9EkLbruJElJC9dRlg==", + "dev": true, + "dependencies": { + "core-js": "^1.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "node_modules/base62": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/base62/-/base62-1.2.8.tgz", + "integrity": "sha512-V6YHUbjLxN1ymqNLb1DPHoU1CpfdL7d2YTIp5W3U4hhoG4hhxNmsFDs66M9EXxBiSEke5Bt5dwdfMwwZF70iLA==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/commoner": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/commoner/-/commoner-0.10.8.tgz", + "integrity": "sha512-3/qHkNMM6o/KGXHITA14y78PcfmXh4+AOCJpSoF73h4VY1JpdGv3CHMS5+JW6SwLhfJt4RhNmLAa7+RRX/62EQ==", + "dev": true, + "dependencies": { + "commander": "^2.5.0", + "detective": "^4.3.1", + "glob": "^5.0.15", + "graceful-fs": "^4.1.2", + "iconv-lite": "^0.4.5", + "mkdirp": "^0.5.0", + "private": "^0.1.6", + "q": "^1.1.2", + "recast": "^0.11.17" + }, + "bin": { + "commonize": "bin/commonize" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "node_modules/core-js": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", + "integrity": "sha512-ZiPp9pZlgxpWRu0M+YWbm6+aQ84XEfH1JRXvfOc/fILWI0VKhLC2LX13X1NYq4fULzLMq7Hfh43CSo2/aIaUPA==", + "deprecated": "core-js@<3.4 is no longer maintained and not recommended for usage due to the number of issues. Because of the V8 engine whims, feature detection in old core-js versions could cause a slowdown up to 100x even if nothing is polyfilled. Please, upgrade your dependencies to the actual version of core-js.", + "dev": true + }, + "node_modules/create-react-class": { + "version": "15.7.0", + "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.7.0.tgz", + "integrity": "sha512-QZv4sFWG9S5RUvkTYWbflxeZX+JG7Cz0Tn33rQBJ+WFQTqTfUTjMjiv9tnfXazjsO5r0KhPs+AqCjyrQX6h2ng==", + "dev": true, + "dependencies": { + "loose-envify": "^1.3.1", + "object-assign": "^4.1.1" + } + }, + "node_modules/create-react-class/node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/defined": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", + "integrity": "sha512-Y2caI5+ZwS5c3RiNDJ6u53VhQHv+hHKwhkI1iHvceKUHw9Df6EK2zRLfjejRgMuCuxK7PfSWIMwWecceVvThjQ==", + "dev": true + }, + "node_modules/detective": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz", + "integrity": "sha512-H6PmeeUcZloWtdt4DAkFyzFL94arpHr3NOwwmVILFiy+9Qd4JTxxXrzfyGk/lmct2qVGBwTSwSXagqu2BxmWig==", + "dev": true, + "dependencies": { + "acorn": "^5.2.1", + "defined": "^1.0.0" + } + }, + "node_modules/envify": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/envify/-/envify-3.4.1.tgz", + "integrity": "sha512-XLiBFsLtNF0MOZl+vWU59yPb3C2JtrQY2CNJn22KH75zPlHWY5ChcAQuf4knJeWT/lLkrx3sqvhP/J349bt4Bw==", + "dev": true, + "dependencies": { + "jstransform": "^11.0.3", + "through": "~2.3.4" + }, + "bin": { + "envify": "bin/envify" + } + }, + "node_modules/esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha512-AWwVMNxwhN8+NIPQzAQZCm7RkLC4RbM3B1OobMuyp3i+w73X57KCKaVIxaRZb+DYCojq7rspo+fmuQfAboyhFg==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esprima-fb": { + "version": "15001.1.0-dev-harmony-fb", + "resolved": "https://registry.npmjs.org/esprima-fb/-/esprima-fb-15001.1.0-dev-harmony-fb.tgz", + "integrity": "sha512-59dDGQo2b3M/JfKIws0/z8dcXH2mnVHkfSPRhCYS91JNGfGNwr7GsSF6qzWZuOGvw5Ii0w9TtylrX07MGmlOoQ==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/fbjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.6.1.tgz", + "integrity": "sha512-4KW7tT33ytfazK3Ekvesbsa4A5J79hUrdXONQGZ0wM6i3PFc70YknF9kj1eyx3mDupgJ7Z+ifFhcMJ+ps2eZIw==", + "dev": true, + "dependencies": { + "core-js": "^1.0.0", + "loose-envify": "^1.0.0", + "promise": "^7.0.3", + "ua-parser-js": "^0.7.9", + "whatwg-fetch": "^0.9.0" + } + }, + "node_modules/glob": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", + "integrity": "sha512-c9IPMazfRITpmAAKi22dK1VKxGDX9ehhqfABDriL/lzO92xcUKEJPQHrVA/2YHSNFB4iFlykVmWvwo48nr3OxA==", + "dev": true, + "dependencies": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "dev": true, + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dev": true, + "dependencies": { + "loose-envify": "^1.0.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/jstransform": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/jstransform/-/jstransform-11.0.3.tgz", + "integrity": "sha512-LGm87w0A8E92RrcXt94PnNHkFqHmgDy3mKHvNZOG7QepKCTCH/VB6S+IEN+bT4uLN3gVpOT0vvOOVd96osG71g==", + "dev": true, + "dependencies": { + "base62": "^1.1.0", + "commoner": "^0.10.1", + "esprima-fb": "^15001.1.0-dev-harmony-fb", + "object-assign": "^2.0.0", + "source-map": "^0.4.2" + }, + "bin": { + "jstransform": "bin/jstransform" + }, + "engines": { + "node": ">=0.8.8" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "node_modules/lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", + "dev": true + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dev": true, + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "dev": true + }, + "node_modules/mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, + "dependencies": { + "minimist": "^1.2.6" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/object-assign": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-2.1.1.tgz", + "integrity": "sha512-CdsOUYIh5wIiozhJ3rLQgmUTgcyzFwZZrqhkKhODMoGtPKM+wt0h0CNIoauJWMsS9822EdzPsF/6mb4nLvPN5g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/private": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", + "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "dev": true, + "dependencies": { + "asap": "~2.0.3" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "dev": true, + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/prop-types/node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", + "dev": true, + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } + }, + "node_modules/react": { + "version": "0.14.10", + "resolved": "https://registry.npmjs.org/react/-/react-0.14.10.tgz", + "integrity": "sha512-yxMw5aorZG4qsLVBfjae4wGFvd5708DhcxaXLJ3IOTgr1TCs8k9+ZheGgLGr5OfwWMhSahNbGvvoEDzrxVWouA==", + "dev": true, + "dependencies": { + "envify": "^3.0.0", + "fbjs": "^0.6.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "dev": true + }, + "node_modules/react-json-tree": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/react-json-tree/-/react-json-tree-0.1.9.tgz", + "integrity": "sha512-GRodIkv0qcTpEY292EeeDeukDbW/RROb3s/8RjnmgnAyPKWQj9thym1l6ly2py4/7rUSQ34UvsK4BW6BmA5ZlQ==", + "dev": true, + "dependencies": { + "babel-runtime": "^5.8.20", + "react-mixin": "^1.7.0" + }, + "peerDependencies": { + "react": ">=0.13.3 || ^0.14.0-beta3" + } + }, + "node_modules/react-mixin": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/react-mixin/-/react-mixin-1.7.0.tgz", + "integrity": "sha512-PZzt6FGoNg2vQPo2pNb84CI0k6s01ZUESHuj1rDTr9ngTvnuMjFmyMibrmpRLyU7enzgbN+vcBfDxX0cXeXIMg==", + "dev": true, + "dependencies": { + "object-assign": "^2.0.0", + "smart-mixin": "^1.2.0" + } + }, + "node_modules/react-redux": { + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-4.4.10.tgz", + "integrity": "sha512-tjL0Bmpkj75Td0k+lXlF8Fc8a9GuXFv/3ahUOCXExWs/jhsKiQeTffdH0j5byejCGCRL4tvGFYlrwBF1X/Aujg==", + "dev": true, + "dependencies": { + "create-react-class": "^15.5.1", + "hoist-non-react-statics": "^3.3.0", + "invariant": "^2.0.0", + "lodash": "^4.17.11", + "loose-envify": "^1.4.0", + "prop-types": "^15.7.2" + }, + "peerDependencies": { + "react": "^0.14.0 || ^15.0.0-0 || ^15.4.0-0 || ^16.0.0-0", + "redux": "^2.0.0 || ^3.0.0" + } + }, + "node_modules/recast": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.11.23.tgz", + "integrity": "sha512-+nixG+3NugceyR8O1bLU45qs84JgI3+8EauyRZafLgC9XbdAOIVgwV1Pe2da0YzGo62KzWoZwUpVEQf6qNAXWA==", + "dev": true, + "dependencies": { + "ast-types": "0.9.6", + "esprima": "~3.1.0", + "private": "~0.1.5", + "source-map": "~0.5.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/recast/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/redux": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/redux/-/redux-3.7.2.tgz", + "integrity": "sha512-pNqnf9q1hI5HHZRBkj3bAngGZW/JMCmexDlOxw4XagXY2o1327nHH54LoTjiPJ0gizoqPDRqWyX/00g0hD6w+A==", + "dev": true, + "dependencies": { + "lodash": "^4.2.1", + "lodash-es": "^4.2.1", + "loose-envify": "^1.1.0", + "symbol-observable": "^1.0.3" + } + }, + "node_modules/redux-devtools": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/redux-devtools/-/redux-devtools-2.1.5.tgz", + "integrity": "sha512-W4vWtyF0+PAGcBNLrzUrYwhz6SayEsZMcWr2BW9WBTAwd9f0Bb1tXB1+Y8LYxvyMqKgTJuFEPDVm2uc2T7OH+w==", + "deprecated": "Package moved to @redux-devtools/core.", + "dev": true, + "dependencies": { + "react-json-tree": "^0.1.9", + "react-mixin": "^1.7.0", + "react-redux": "^3.0.0", + "redux": "^2.0.0 || ^3.0.0" + }, + "peerDependencies": { + "redux": "^2.0.0 || ^3.0.0" + } + }, + "node_modules/redux-devtools/node_modules/hoist-non-react-statics": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-1.2.0.tgz", + "integrity": "sha512-r8huvKK+m+VraiRipdZYc+U4XW43j6OFG/oIafe7GfDbRpCduRoX9JI/DRxqgtBSCeL+et6N6ibZoedHS2NyOQ==", + "dev": true + }, + "node_modules/redux-devtools/node_modules/react-redux": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-3.1.2.tgz", + "integrity": "sha512-+cMUkkZhwW82YawY3uaZc45g3WfAA7aaPJp87lUkkQ7V1DGypfU5P/suNmGiZTO8x8SDyOV8+GEQ2bcEOx/K4w==", + "dev": true, + "dependencies": { + "hoist-non-react-statics": "^1.0.3", + "invariant": "^2.0.0" + }, + "peerDependencies": { + "redux": "^2.0.0 || ^3.0.0" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/smart-mixin": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/smart-mixin/-/smart-mixin-1.2.1.tgz", + "integrity": "sha512-PMHIQCtRq7h6pZAw0p8o8WuUyiK+sxsso4Czrh+zLdoEvuep+iRfQ4fL0PWI7Lq8K2ZfwgswvZ39hV8uJZ2qiw==", + "dev": true + }, + "node_modules/snyk": { + "version": "1.952.0", + "resolved": "https://registry.npmjs.org/snyk/-/snyk-1.952.0.tgz", + "integrity": "sha512-aE3eUF0psAKyNvq8WIGQ57onaxZCIpHueXivVIzJBOsVGbUKJAEGfg5BI1VGfOUqoZKtdgj1mUxDNHjuEGWFZQ==", + "dev": true, + "bin": { + "snyk": "bin/snyk" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", + "dev": true, + "dependencies": { + "amdefine": ">=0.0.4" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "node_modules/ua-parser-js": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.31.tgz", + "integrity": "sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/ua-parser-js" + }, + { + "type": "paypal", + "url": "https://paypal.me/faisalman" + } + ], + "engines": { + "node": "*" + } + }, + "node_modules/whatwg-fetch": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-0.9.0.tgz", + "integrity": "sha512-DIuh7/cloHxHYwS/oRXGgkALYAntijL63nsgMQsNSnBj825AysosAqA2ZbYXGRqpPRiNH7335dTqV364euRpZw==", + "dev": true + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + } + }, + "dependencies": { + "acorn": { + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", + "dev": true + }, + "amdefine": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", + "integrity": "sha512-S2Hw0TtNkMJhIabBwIojKL9YHO5T0n5eNqWJ7Lrlel/zDbftQpxpapi8tZs3X1HWa+u+QeydGmzzNU0m09+Rcg==", + "dev": true + }, + "asap": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/asap/-/asap-2.0.6.tgz", + "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==", + "dev": true + }, + "ast-types": { + "version": "0.9.6", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.9.6.tgz", + "integrity": "sha512-qEdtR2UH78yyHX/AUNfXmJTlM48XoFZKBdwi1nzkI1mJL21cmbu0cvjxjpkXJ5NENMq42H+hNs8VLJcqXLerBQ==", + "dev": true + }, + "babel-runtime": { + "version": "5.8.38", + "resolved": "https://registry.npmjs.org/babel-runtime/-/babel-runtime-5.8.38.tgz", + "integrity": "sha512-KpgoA8VE/pMmNCrnEeeXqFG24TIH11Z3ZaimIhJWsin8EbfZy3WzFKUTIan10ZIDgRVvi9EkLbruJElJC9dRlg==", + "dev": true, + "requires": { + "core-js": "^1.0.0" + } + }, + "balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true + }, + "base62": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/base62/-/base62-1.2.8.tgz", + "integrity": "sha512-V6YHUbjLxN1ymqNLb1DPHoU1CpfdL7d2YTIp5W3U4hhoG4hhxNmsFDs66M9EXxBiSEke5Bt5dwdfMwwZF70iLA==", + "dev": true + }, + "brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "commoner": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/commoner/-/commoner-0.10.8.tgz", + "integrity": "sha512-3/qHkNMM6o/KGXHITA14y78PcfmXh4+AOCJpSoF73h4VY1JpdGv3CHMS5+JW6SwLhfJt4RhNmLAa7+RRX/62EQ==", + "dev": true, + "requires": { + "commander": "^2.5.0", + "detective": "^4.3.1", + "glob": "^5.0.15", + "graceful-fs": "^4.1.2", + "iconv-lite": "^0.4.5", + "mkdirp": "^0.5.0", + "private": "^0.1.6", + "q": "^1.1.2", + "recast": "^0.11.17" + } + }, + "concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true + }, + "core-js": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-1.2.7.tgz", + "integrity": "sha512-ZiPp9pZlgxpWRu0M+YWbm6+aQ84XEfH1JRXvfOc/fILWI0VKhLC2LX13X1NYq4fULzLMq7Hfh43CSo2/aIaUPA==", + "dev": true + }, + "create-react-class": { + "version": "15.7.0", + "resolved": "https://registry.npmjs.org/create-react-class/-/create-react-class-15.7.0.tgz", + "integrity": "sha512-QZv4sFWG9S5RUvkTYWbflxeZX+JG7Cz0Tn33rQBJ+WFQTqTfUTjMjiv9tnfXazjsO5r0KhPs+AqCjyrQX6h2ng==", + "dev": true, + "requires": { + "loose-envify": "^1.3.1", + "object-assign": "^4.1.1" + }, + "dependencies": { + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true + } + } + }, + "defined": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.0.tgz", + "integrity": "sha512-Y2caI5+ZwS5c3RiNDJ6u53VhQHv+hHKwhkI1iHvceKUHw9Df6EK2zRLfjejRgMuCuxK7PfSWIMwWecceVvThjQ==", + "dev": true + }, + "detective": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/detective/-/detective-4.7.1.tgz", + "integrity": "sha512-H6PmeeUcZloWtdt4DAkFyzFL94arpHr3NOwwmVILFiy+9Qd4JTxxXrzfyGk/lmct2qVGBwTSwSXagqu2BxmWig==", + "dev": true, + "requires": { + "acorn": "^5.2.1", + "defined": "^1.0.0" + } + }, + "envify": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/envify/-/envify-3.4.1.tgz", + "integrity": "sha512-XLiBFsLtNF0MOZl+vWU59yPb3C2JtrQY2CNJn22KH75zPlHWY5ChcAQuf4knJeWT/lLkrx3sqvhP/J349bt4Bw==", + "dev": true, + "requires": { + "jstransform": "^11.0.3", + "through": "~2.3.4" + } + }, + "esprima": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-3.1.3.tgz", + "integrity": "sha512-AWwVMNxwhN8+NIPQzAQZCm7RkLC4RbM3B1OobMuyp3i+w73X57KCKaVIxaRZb+DYCojq7rspo+fmuQfAboyhFg==", + "dev": true + }, + "esprima-fb": { + "version": "15001.1.0-dev-harmony-fb", + "resolved": "https://registry.npmjs.org/esprima-fb/-/esprima-fb-15001.1.0-dev-harmony-fb.tgz", + "integrity": "sha512-59dDGQo2b3M/JfKIws0/z8dcXH2mnVHkfSPRhCYS91JNGfGNwr7GsSF6qzWZuOGvw5Ii0w9TtylrX07MGmlOoQ==", + "dev": true + }, + "fbjs": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/fbjs/-/fbjs-0.6.1.tgz", + "integrity": "sha512-4KW7tT33ytfazK3Ekvesbsa4A5J79hUrdXONQGZ0wM6i3PFc70YknF9kj1eyx3mDupgJ7Z+ifFhcMJ+ps2eZIw==", + "dev": true, + "requires": { + "core-js": "^1.0.0", + "loose-envify": "^1.0.0", + "promise": "^7.0.3", + "ua-parser-js": "^0.7.9", + "whatwg-fetch": "^0.9.0" + } + }, + "glob": { + "version": "5.0.15", + "resolved": "https://registry.npmjs.org/glob/-/glob-5.0.15.tgz", + "integrity": "sha512-c9IPMazfRITpmAAKi22dK1VKxGDX9ehhqfABDriL/lzO92xcUKEJPQHrVA/2YHSNFB4iFlykVmWvwo48nr3OxA==", + "dev": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "graceful-fs": { + "version": "4.2.10", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", + "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", + "dev": true + }, + "hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "dev": true, + "requires": { + "react-is": "^16.7.0" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "dev": true, + "requires": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "invariant": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/invariant/-/invariant-2.2.4.tgz", + "integrity": "sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA==", + "dev": true, + "requires": { + "loose-envify": "^1.0.0" + } + }, + "js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "jstransform": { + "version": "11.0.3", + "resolved": "https://registry.npmjs.org/jstransform/-/jstransform-11.0.3.tgz", + "integrity": "sha512-LGm87w0A8E92RrcXt94PnNHkFqHmgDy3mKHvNZOG7QepKCTCH/VB6S+IEN+bT4uLN3gVpOT0vvOOVd96osG71g==", + "dev": true, + "requires": { + "base62": "^1.1.0", + "commoner": "^0.10.1", + "esprima-fb": "^15001.1.0-dev-harmony-fb", + "object-assign": "^2.0.0", + "source-map": "^0.4.2" + } + }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", + "dev": true + }, + "lodash-es": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", + "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==", + "dev": true + }, + "loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "dev": true, + "requires": { + "js-tokens": "^3.0.0 || ^4.0.0" + } + }, + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + }, + "minimist": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", + "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "dev": true + }, + "mkdirp": { + "version": "0.5.6", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", + "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", + "dev": true, + "requires": { + "minimist": "^1.2.6" + } + }, + "object-assign": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-2.1.1.tgz", + "integrity": "sha512-CdsOUYIh5wIiozhJ3rLQgmUTgcyzFwZZrqhkKhODMoGtPKM+wt0h0CNIoauJWMsS9822EdzPsF/6mb4nLvPN5g==", + "dev": true + }, + "once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "requires": { + "wrappy": "1" + } + }, + "path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true + }, + "private": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/private/-/private-0.1.8.tgz", + "integrity": "sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==", + "dev": true + }, + "promise": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", + "integrity": "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg==", + "dev": true, + "requires": { + "asap": "~2.0.3" + } + }, + "prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "dev": true, + "requires": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + }, + "dependencies": { + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "dev": true + } + } + }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", + "dev": true + }, + "react": { + "version": "0.14.10", + "resolved": "https://registry.npmjs.org/react/-/react-0.14.10.tgz", + "integrity": "sha512-yxMw5aorZG4qsLVBfjae4wGFvd5708DhcxaXLJ3IOTgr1TCs8k9+ZheGgLGr5OfwWMhSahNbGvvoEDzrxVWouA==", + "dev": true, + "requires": { + "envify": "^3.0.0", + "fbjs": "^0.6.1" + } + }, + "react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "dev": true + }, + "react-json-tree": { + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/react-json-tree/-/react-json-tree-0.1.9.tgz", + "integrity": "sha512-GRodIkv0qcTpEY292EeeDeukDbW/RROb3s/8RjnmgnAyPKWQj9thym1l6ly2py4/7rUSQ34UvsK4BW6BmA5ZlQ==", + "dev": true, + "requires": { + "babel-runtime": "^5.8.20", + "react-mixin": "^1.7.0" + } + }, + "react-mixin": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/react-mixin/-/react-mixin-1.7.0.tgz", + "integrity": "sha512-PZzt6FGoNg2vQPo2pNb84CI0k6s01ZUESHuj1rDTr9ngTvnuMjFmyMibrmpRLyU7enzgbN+vcBfDxX0cXeXIMg==", + "dev": true, + "requires": { + "object-assign": "^2.0.0", + "smart-mixin": "^1.2.0" + } + }, + "react-redux": { + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-4.4.10.tgz", + "integrity": "sha512-tjL0Bmpkj75Td0k+lXlF8Fc8a9GuXFv/3ahUOCXExWs/jhsKiQeTffdH0j5byejCGCRL4tvGFYlrwBF1X/Aujg==", + "dev": true, + "requires": { + "create-react-class": "^15.5.1", + "hoist-non-react-statics": "^3.3.0", + "invariant": "^2.0.0", + "lodash": "^4.17.11", + "loose-envify": "^1.4.0", + "prop-types": "^15.7.2" + } + }, + "recast": { + "version": "0.11.23", + "resolved": "https://registry.npmjs.org/recast/-/recast-0.11.23.tgz", + "integrity": "sha512-+nixG+3NugceyR8O1bLU45qs84JgI3+8EauyRZafLgC9XbdAOIVgwV1Pe2da0YzGo62KzWoZwUpVEQf6qNAXWA==", + "dev": true, + "requires": { + "ast-types": "0.9.6", + "esprima": "~3.1.0", + "private": "~0.1.5", + "source-map": "~0.5.0" + }, + "dependencies": { + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "dev": true + } + } + }, + "redux": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/redux/-/redux-3.7.2.tgz", + "integrity": "sha512-pNqnf9q1hI5HHZRBkj3bAngGZW/JMCmexDlOxw4XagXY2o1327nHH54LoTjiPJ0gizoqPDRqWyX/00g0hD6w+A==", + "dev": true, + "requires": { + "lodash": "^4.2.1", + "lodash-es": "^4.2.1", + "loose-envify": "^1.1.0", + "symbol-observable": "^1.0.3" + } + }, + "redux-devtools": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/redux-devtools/-/redux-devtools-2.1.5.tgz", + "integrity": "sha512-W4vWtyF0+PAGcBNLrzUrYwhz6SayEsZMcWr2BW9WBTAwd9f0Bb1tXB1+Y8LYxvyMqKgTJuFEPDVm2uc2T7OH+w==", + "dev": true, + "requires": { + "react-json-tree": "^0.1.9", + "react-mixin": "^1.7.0", + "react-redux": "^3.0.0", + "redux": "^2.0.0 || ^3.0.0" + }, + "dependencies": { + "hoist-non-react-statics": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-1.2.0.tgz", + "integrity": "sha512-r8huvKK+m+VraiRipdZYc+U4XW43j6OFG/oIafe7GfDbRpCduRoX9JI/DRxqgtBSCeL+et6N6ibZoedHS2NyOQ==", + "dev": true + }, + "react-redux": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-3.1.2.tgz", + "integrity": "sha512-+cMUkkZhwW82YawY3uaZc45g3WfAA7aaPJp87lUkkQ7V1DGypfU5P/suNmGiZTO8x8SDyOV8+GEQ2bcEOx/K4w==", + "dev": true, + "requires": { + "hoist-non-react-statics": "^1.0.3", + "invariant": "^2.0.0" + } + } + } + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "smart-mixin": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/smart-mixin/-/smart-mixin-1.2.1.tgz", + "integrity": "sha512-PMHIQCtRq7h6pZAw0p8o8WuUyiK+sxsso4Czrh+zLdoEvuep+iRfQ4fL0PWI7Lq8K2ZfwgswvZ39hV8uJZ2qiw==", + "dev": true + }, + "snyk": { + "version": "1.952.0", + "resolved": "https://registry.npmjs.org/snyk/-/snyk-1.952.0.tgz", + "integrity": "sha512-aE3eUF0psAKyNvq8WIGQ57onaxZCIpHueXivVIzJBOsVGbUKJAEGfg5BI1VGfOUqoZKtdgj1mUxDNHjuEGWFZQ==", + "dev": true + }, + "source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha512-Y8nIfcb1s/7DcobUz1yOO1GSp7gyL+D9zLHDehT7iRESqGSxjJ448Sg7rvfgsRJCnKLdSl11uGf0s9X80cH0/A==", + "dev": true, + "requires": { + "amdefine": ">=0.0.4" + } + }, + "symbol-observable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.2.0.tgz", + "integrity": "sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==", + "dev": true + }, + "through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==", + "dev": true + }, + "ua-parser-js": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.31.tgz", + "integrity": "sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ==", + "dev": true + }, + "whatwg-fetch": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-0.9.0.tgz", + "integrity": "sha512-DIuh7/cloHxHYwS/oRXGgkALYAntijL63nsgMQsNSnBj825AysosAqA2ZbYXGRqpPRiNH7335dTqV364euRpZw==", + "dev": true + }, + "wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true + } + } +} diff --git a/politician/models.py b/politician/models.py index 13bb86e6c..6c5639c5f 100644 --- a/politician/models.py +++ b/politician/models.py @@ -1244,7 +1244,7 @@ def retrieve_politicians_are_not_duplicates_list(self, politician_we_vote_id, re } return results - def retrieve_politicians_with_misformatted_names(self, start=0, count=15): + def retrieve_politicians_with_misformatted_names(self, start=0, count=15, read_only=False): """ Get the first 15 records that have 3 capitalized letters in a row, as long as those letters are not 'III' i.e. King Henry III. Also exclude the names where the word "WITHDRAWN" has been appended when @@ -1253,9 +1253,14 @@ def retrieve_politicians_with_misformatted_names(self, start=0, count=15): politician_name !~ '.*?III.*?' :param start: + :param count: + :param read_only: :return: """ - politician_query = Politician.objects.all() + if positive_value_exists(read_only): + politician_query = Politician.objects.using('readonly').all() + else: + politician_query = Politician.objects.all() # Get all politicians that have three capital letters in a row in their name, but exclude III (King Henry III) politician_query = politician_query.filter(politician_name__regex=r'.*?[A-Z][A-Z][A-Z].*?(? NUMBER_OF_MAP_POINTS_THRESHOLD_FOR_EXTRA_LARGE_STATE: # 14000 + retrieved_each_batch_chunk = MAP_POINTS_RETRIEVED_EACH_BATCH_CHUNK_FOR_EXTRA_LARGE_STATE # 51 + elif number_of_polling_locations > NUMBER_OF_MAP_POINTS_THRESHOLD_FOR_LARGE_STATE: # 9000 + retrieved_each_batch_chunk = MAP_POINTS_RETRIEVED_EACH_BATCH_CHUNK_FOR_LARGE_STATE # 95 + + return retrieved_each_batch_chunk + def fetch_polling_location_count( self, state_code=''): @@ -481,7 +505,7 @@ def soft_delete_polling_location_log_entry_list( success = True number_deleted = 0 try: - query = PollingLocationLogEntry.objects.all() + query = PollingLocationLogEntry.objects.using('analytics').all() query = query.exclude(log_entry_deleted=True) if positive_value_exists(len(kind_of_log_entry_list) > 0): query = query.filter(kind_of_log_entry__in=kind_of_log_entry_list) @@ -932,10 +956,11 @@ def retrieve_polling_location_log_entry_list( polling_location_log_entry_list_found = False polling_location_log_entry_list = [] try: - if positive_value_exists(read_only): - query = PollingLocationLogEntry.objects.using('readonly').all() - else: - query = PollingLocationLogEntry.objects.all() + # if positive_value_exists(read_only): + # query = PollingLocationLogEntry.objects.using('readonly').all() + # else: + # query = PollingLocationLogEntry.objects.using('analytics').all() + query = PollingLocationLogEntry.objects.using('analytics').all() if positive_value_exists(batch_process_id): query = query.filter(batch_process_id=batch_process_id) if positive_value_exists(google_civic_election_id): diff --git a/polling_location/views_admin.py b/polling_location/views_admin.py index cec622e6c..9dabd6096 100644 --- a/polling_location/views_admin.py +++ b/polling_location/views_admin.py @@ -521,6 +521,19 @@ def polling_location_list_view(request): polling_location_count_query = PollingLocation.objects.all() polling_location_without_latitude_count = 0 polling_location_query = PollingLocation.objects.all() + selected_types = [] + filtered_polling_locations = [] + + polling_location_source_codes = set([polling_location.source_code for polling_location in polling_location_query]) + + for polling_location_source_code in polling_location_source_codes: + if request.GET.get(f'show_{polling_location_source_code}', 0): + selected_types.append(polling_location_source_code) + + if selected_types: + polling_location_count_query = polling_location_count_query.filter(source_code__in = selected_types) + polling_location_query = polling_location_query.filter(source_code__in = selected_types) + if not positive_value_exists(polling_location_search): polling_location_count_query = polling_location_count_query.exclude(polling_location_deleted=True) polling_location_query = polling_location_query.exclude(polling_location_deleted=True) @@ -688,6 +701,8 @@ def polling_location_list_view(request): 'show_successful_retrieves': show_successful_retrieves, 'show_vote_usa_errors': show_vote_usa_errors, 'state_code': state_code, + 'selected_types': selected_types, + 'source_code_list': polling_location_source_codes, 'state_name': convert_state_code_to_state_text(state_code), 'state_list': sorted_state_list, } diff --git a/position/controllers.py b/position/controllers.py index 10964c8cd..4e20a7e4f 100644 --- a/position/controllers.py +++ b/position/controllers.py @@ -355,7 +355,7 @@ def generate_position_sorting_dates_for_election(google_civic_election_id=0): loop_number += 1 try: # Get a list of candidate_we_vote_ids for this election which haven't been updated yet - query = CandidateToOfficeLink.objects.all() + query = CandidateToOfficeLink.objects.using('readonly').all() query = query.filter(google_civic_election_id=google_civic_election_id) query = query.filter(position_dates_set=False) query = query.values_list('candidate_we_vote_id', flat=True).distinct() @@ -2063,9 +2063,9 @@ def position_list_for_ballot_item_for_api(office_id, office_we_vote_id, # posit # the WebApp team) candidate_manager = CandidateManager() if positive_value_exists(candidate_id): - results = candidate_manager.retrieve_candidate_from_id(candidate_id) + results = candidate_manager.retrieve_candidate_from_id(candidate_id, read_only=True) else: - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] @@ -2302,10 +2302,12 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot results = is_voter_device_id_valid(voter_device_id) if not results['success']: position_list = [] + status += 'VALID_VOTER_DEVICE_ID_MISSING ' json_data = { - 'status': 'VALID_VOTER_DEVICE_ID_MISSING', + 'status': status, 'success': False, 'count': 0, + 'friends_vs_public': friends_vs_public, 'kind_of_ballot_item': "UNKNOWN", 'ballot_item_id': 0, 'position_list': position_list, @@ -2313,7 +2315,7 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot return HttpResponse(json.dumps(json_data), content_type='application/json') voter_manager = VoterManager() - voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id) + voter_results = voter_manager.retrieve_voter_from_voter_device_id(voter_device_id, read_only=True) if voter_results['voter_found']: voter = voter_results['voter'] voter_id = voter.id @@ -2323,10 +2325,12 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot voter_we_vote_id = "" if not positive_value_exists(voter_id): position_list = [] + status += "VALID_VOTER_ID_MISSING " json_data = { - 'status': "VALID_VOTER_ID_MISSING ", + 'status': status, 'success': False, 'count': 0, + 'friends_vs_public': friends_vs_public, 'kind_of_ballot_item': "UNKNOWN", 'ballot_item_id': 0, 'position_list': position_list, @@ -2457,6 +2461,7 @@ def position_list_for_ballot_item_from_friends_for_api( # positionListForBallot 'status': status, 'success': success, 'count': positions_count, + 'friends_vs_public': friends_vs_public, 'kind_of_ballot_item': kind_of_ballot_item, 'ballot_item_id': ballot_item_id, 'ballot_item_we_vote_id': ballot_item_we_vote_id, @@ -2540,9 +2545,9 @@ def retrieve_position_list_for_ballot_item_from_friends( # the WebApp team) candidate_manager = CandidateManager() if positive_value_exists(candidate_id): - results = candidate_manager.retrieve_candidate_from_id(candidate_id) + results = candidate_manager.retrieve_candidate_from_id(candidate_id, read_only=True) else: - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] @@ -2828,9 +2833,9 @@ def retrieve_position_list_for_ballot_item_from_shared_items( # the WebApp team) candidate_manager = CandidateManager() if positive_value_exists(candidate_id): - results = candidate_manager.retrieve_candidate_from_id(candidate_id) + results = candidate_manager.retrieve_candidate_from_id(candidate_id, read_only=True) else: - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] @@ -4465,7 +4470,8 @@ def refresh_positions_with_candidate_details_for_election(google_civic_election_ candidates_results = candidate_list_manager.retrieve_all_candidates_for_upcoming_election( google_civic_election_id_list=google_civic_election_id_list, state_code=state_code, - return_list_of_objects=return_list_of_objects) + return_list_of_objects=return_list_of_objects, + read_only=True) if candidates_results['candidate_list_found']: candidate_list = candidates_results['candidate_list_objects'] diff --git a/position/models.py b/position/models.py index b8aa21fc3..c2af76092 100644 --- a/position/models.py +++ b/position/models.py @@ -1462,242 +1462,218 @@ def repair_all_positions_for_voter(self, incoming_voter_id=0, incoming_voter_we_ failure_counter = 0 ############################ - # Retrieve public positions + # Repair positions owned by voter_id + public_number_changed = 0 + friend_number_changed = 0 try: - # Retrieve by voter_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter(voter_id=voter_id) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() - - except Exception as e: - failure_counter += 1 - - # Retrieve by voter_we_vote_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter( - voter_we_vote_id__iexact=voter_we_vote_id) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() - - except Exception as e: - failure_counter += 1 - - # Retrieve by organization_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter(organization_id=organization_id) - # As of Aug 2018 we are no longer using PERCENT_RATING - public_positions_list_query = public_positions_list_query.exclude(stance__iexact=PERCENT_RATING) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() - - except Exception as e: - failure_counter += 1 - - # Retrieve by organization_we_vote_id - public_positions_list_query = PositionEntered.objects.all() - public_positions_list_query = public_positions_list_query.filter( - organization_we_vote_id__iexact=organization_we_vote_id) - # As of Aug 2018 we are no longer using PERCENT_RATING - public_positions_list_query = public_positions_list_query.exclude(stance__iexact=PERCENT_RATING) - public_positions_list = list(public_positions_list_query) # Force the query to run - for public_position in public_positions_list: - public_position_to_be_saved = False - try: - if public_position.voter_id != voter_id: - public_position.voter_id = voter_id - public_position_to_be_saved = True - if public_position.voter_we_vote_id != voter_we_vote_id: - public_position.voter_we_vote_id = voter_we_vote_id - public_position_to_be_saved = True - if public_position.organization_id != organization_id: - public_position.organization_id = organization_id - public_position_to_be_saved = True - if public_position.organization_we_vote_id != organization_we_vote_id: - public_position.organization_we_vote_id = organization_we_vote_id - public_position_to_be_saved = True - - if public_position_to_be_saved: - public_position.save() + # ...without the right voter_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_id=voter_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_id=voter_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right organization_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + # ...without the right organization_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_id=voter_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) except Exception as e: - results = { - 'status': 'REPAIR-VOTER_POSITION_FOR_PUBLIC_SEARCH_FAILED ', - 'success': False, - 'repair_complete': False, - } - return results + failure_counter += 1 ############################ - # Retrieve positions meant for friends only + # Repair public positions owned by voter_we_vote_id try: - # Retrieve by voter_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter(voter_id=voter_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + # ...without the right voter_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right organization_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) - # Retrieve by voter_we_vote_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter( - voter_we_vote_id__iexact=voter_we_vote_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + # ...without the right organization_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + voter_we_vote_id__iexact=voter_we_vote_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + except Exception as e: + failure_counter += 1 - except Exception as e: - failure_counter += 1 + ############################ + # Repair public positions owned by organization_id + try: + # ...without the right voter_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) - # Retrieve by organization_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter(organization_id=organization_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + # ...without the right voter_we_vot_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_id=organization_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right organization_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_id=organization_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_id=organization_id, + ).exclude( + organization_we_vote_id__iexact=organization_we_vote_id, + ).update( + organization_we_vote_id=organization_we_vote_id, + ) + except Exception as e: + failure_counter += 1 - # Retrieve by organization_we_vote_id - friends_positions_list_query = PositionForFriends.objects.all() - friends_positions_list_query = friends_positions_list_query.filter( - organization_we_vote_id__iexact=organization_we_vote_id) - friends_positions_list = list(friends_positions_list_query) # Force the query to run - for friends_position in friends_positions_list: - friends_position_to_be_saved = False - try: - if friends_position.voter_id != voter_id: - friends_position.voter_id = voter_id - friends_position_to_be_saved = True - if friends_position.voter_we_vote_id != voter_we_vote_id: - friends_position.voter_we_vote_id = voter_we_vote_id - friends_position_to_be_saved = True - if friends_position.organization_id != organization_id: - friends_position.organization_id = organization_id - friends_position_to_be_saved = True - if friends_position.organization_we_vote_id != organization_we_vote_id: - friends_position.organization_we_vote_id = organization_we_vote_id - friends_position_to_be_saved = True - - if friends_position_to_be_saved: - friends_position.save() + ############################ + # Repair public positions owned by organization_we_vote_id + try: + # ...without the right voter_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_id=voter_id, + ).update( + voter_id=voter_id, + ) - except Exception as e: - failure_counter += 1 + # ...without the right voter_we_vote_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + voter_we_vote_id__iexact=voter_we_vote_id, + ).update( + voter_we_vote_id=voter_we_vote_id, + ) + # ...without the right organization_id should be updated + public_number_changed += PositionEntered.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) + friend_number_changed += PositionForFriends.objects.all().filter( + organization_we_vote_id__iexact=organization_we_vote_id, + ).exclude( + organization_id=organization_id, + ).update( + organization_id=organization_id, + ) except Exception as e: - results = { - 'status': 'REPAIR-VOTER_POSITION_FOR_FRIENDS_SEARCH_FAILED ', - 'success': False, - 'repair_complete': False, - } - return results + failure_counter += 1 if positive_value_exists(failure_counter): results = { @@ -2250,6 +2226,7 @@ def retrieve_all_positions_for_contest_office(self, retrieve_public_positions=Tr candidate_we_vote_id_list = results['candidate_we_vote_id_list'] # Retrieve the support positions for this contest_office_id + position_list = [] position_list_found = False try: if retrieve_public_positions: @@ -2281,11 +2258,11 @@ def retrieve_all_positions_for_contest_office(self, retrieve_public_positions=Tr # for contest_office (like we do for candidate) because we don't have to deal with # PERCENT_RATING data with measures if friends_we_vote_id_list is not False: - # Find positions from friends. Look for we_vote_id case insensitive. + # Find positions from friends. Look for we_vote_id case-insensitive. we_vote_id_filter = Q() for we_vote_id in friends_we_vote_id_list: we_vote_id_filter |= Q(voter_we_vote_id__iexact=we_vote_id) - position_list_query = position_list_query.filter(we_vote_id_filter) + position_list_query = position_list_query.filter(we_vote_id_filter) # Limit to positions in the last x years - currently we are not limiting # position_list = position_list.filter(election_id=election_id) @@ -4312,7 +4289,7 @@ def create_position_for_visibility_change(self, voter_id, contest_office_we_vote if candidate_we_vote_id: candidate_manager = CandidateManager() - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] ballot_item_display_name = candidate.candidate_name @@ -5400,7 +5377,7 @@ def switch_position_visibility(self, existing_position, switch_to_public_positio if positive_value_exists(existing_position.candidate_campaign_we_vote_id): candidate_manager = CandidateManager() results = candidate_manager.retrieve_candidate_from_we_vote_id( - existing_position.candidate_campaign_we_vote_id) + existing_position.candidate_campaign_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] existing_position.google_civic_election_id = \ @@ -6351,7 +6328,7 @@ def update_or_create_position_comment(self, position_we_vote_id, voter_id, voter if positive_value_exists(voter_position_on_stage.candidate_campaign_we_vote_id): results = candidate_manager.retrieve_candidate_from_we_vote_id( - voter_position_on_stage.candidate_campaign_we_vote_id) + voter_position_on_stage.candidate_campaign_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] if not positive_value_exists(voter_position_on_stage.candidate_campaign_id): @@ -6432,7 +6409,7 @@ def update_or_create_position_comment(self, position_we_vote_id, voter_id, voter speaker_image_url_https_medium = '', speaker_image_url_https_tiny = '', if candidate_we_vote_id: - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] candidate_id = candidate.id @@ -7036,7 +7013,7 @@ def update_or_create_position( position_on_stage.candidate_campaign_we_vote_id = candidate_we_vote_id # Lookup candidate_campaign_id based on candidate_campaign_we_vote_id and update candidate_results = \ - candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if candidate_results['candidate_found']: candidate = candidate_results['candidate'] position_on_stage.candidate_campaign_id = candidate.id @@ -7552,7 +7529,7 @@ def update_or_create_position( candidate_id = None if candidate_we_vote_id: - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] candidate_id = candidate.id @@ -8151,7 +8128,7 @@ def refresh_cached_position_info( candidate_found = True else: candidate_results = candidate_manager.retrieve_candidate_from_we_vote_id( - position_object.candidate_campaign_we_vote_id) + position_object.candidate_campaign_we_vote_id) # Unknown read_only setting needed if candidate_results['candidate_found']: candidate = candidate_results['candidate'] candidate_found = True diff --git a/position/views_admin.py b/position/views_admin.py index d76d1339b..0bb6a4735 100644 --- a/position/views_admin.py +++ b/position/views_admin.py @@ -190,7 +190,7 @@ def update_position_list_with_contest_office_info(position_list): politician_id = candidate.politician_id else: results = candidate_manager.retrieve_candidate_from_we_vote_id( - one_position.candidate_campaign_we_vote_id) + one_position.candidate_campaign_we_vote_id, read_only=False) # May be able to be read_only if results['candidate_found']: candidate = results['candidate'] candidate_dict[one_position.candidate_campaign_we_vote_id] = candidate diff --git a/reaction/models.py b/reaction/models.py index e73954888..f32c4b878 100644 --- a/reaction/models.py +++ b/reaction/models.py @@ -41,7 +41,7 @@ def __unicode__(self): def count_all_reaction_likes(self, liked_item_we_vote_id): # How many people, across the entire network, like this position? try: - reaction_like_query = ReactionLike.objects.all() + reaction_like_query = ReactionLike.objects.using('readonly').all() reaction_like_query = reaction_like_query.filter(liked_item_we_vote_id__iexact=liked_item_we_vote_id) number_of_likes = reaction_like_query.count() status = "REACTION_LIKE_ALL_COUNT_RETRIEVED" @@ -136,7 +136,7 @@ def retrieve_reaction_like_list_for_voter(self, voter_id): def count_voter_network_reaction_likes(self, liked_item_we_vote_id, voter_id): # How many people, limited to the voter's network, like this position? # TODO limit to just the voter's network try: - reaction_like_query = ReactionLike.objects.all() + reaction_like_query = ReactionLike.objects.using('readonly').all() reaction_like_query = reaction_like_query.filter(liked_item_we_vote_id=liked_item_we_vote_id) number_of_likes = reaction_like_query.count() status = "REACTION_LIKE_VOTER_NETWORK_COUNT_RETRIEVED" diff --git a/requirements.txt b/requirements.txt index 0ddbd504f..717addc94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ boto3==1.13.16 cryptography==3.4.8 dj-database-url==0.5.0 dj-static==0.0.6 -django==3.2.13 +django==3.2.15 django-background-tasks==1.2.5 django-bootstrap3==15.0.0 django-cors-headers==3.7.0 @@ -31,14 +31,13 @@ oauthlib==3.1.1 nameparser==1.0.6 phonenumbers==8.12.39 Pillow==9.0.1 -psycopg2-binary==2.8.6 +psycopg2-binary==2.9.3 pyasn1==0.4.8 pyasn1-modules==0.2.8 py3dns==3.2.1 pygeoip==0.3.2 pyjwkest==1.4.2 -# 4/21/22, needed to update for PyJWT==1.7.1 # We are stuck at 1.7.1 until twillio can handle a newer version (as of 6/15/21) -PyJWT==2.4.0 # We are stuck at 1.7.1 until twillio can handle a newer version (as of 6/15/21) +PyJWT==2.4.0 python-magic==0.4.24 # Requires "brew install libmagic" or "brew upgrade libmagic" or "pip install libmagic" python3-openid -e git+git://github.com/wevote/python3-openid.git@master#egg=python3-openid pytz==2021.1 @@ -47,7 +46,8 @@ requests-oauthlib==1.3.0 robot-detection==0.4 rsa==4.7.2 selenium==3.141.0 -sendgrid-django==4.2.0 +sendgrid==6.9.7 +# sendgrid-django==4.2.0 setuptools simplejson==3.17.2 six==1.14.0 diff --git a/retrieve_tables/controllers.py b/retrieve_tables/controllers.py index 9912384aa..825f23063 100644 --- a/retrieve_tables/controllers.py +++ b/retrieve_tables/controllers.py @@ -2,13 +2,15 @@ import json import os import re +import time +from io import StringIO import psycopg2 import requests -import time -from config.base import get_environment_variable from django.http import HttpResponse + import wevote_functions.admin +from config.base import get_environment_variable from wevote_functions.functions import positive_value_exists logger = wevote_functions.admin.get_logger(__name__) @@ -21,12 +23,13 @@ # 'campaign_campaignx_politician', # 'campaign_campaignxseofriendlypath', allowable_tables = [ - 'candidate_candidatecampaign', + 'ballot_ballotitem', + 'position_positionentered', 'candidate_candidatesarenotduplicates', 'candidate_candidatetoofficelink', 'elected_office_electedoffice', 'elected_official_electedofficial', - 'elected_official_electedofficialsarenotduplicates', + 'elected_official_electedofficialsarenotduplicates', # July 2022: This table is empty, why are we loading it? 'election_ballotpediaelection', 'election_election', 'electoral_district_electoraldistrict', @@ -38,37 +41,39 @@ 'office_contestoffice', 'office_contestofficesarenotduplicates', 'office_contestofficevisitingotherelection', - 'organization_organization', 'organization_organizationreserveddomain', 'party_party', 'politician_politician', 'politician_politiciansarenotduplicates', - 'polling_location_pollinglocation', - 'position_positionentered', 'twitter_twitterlinktoorganization', 'voter_guide_voterguidepossibility', 'voter_guide_voterguidepossibilityposition', 'voter_guide_voterguide', 'wevote_settings_wevotesetting', - 'ballot_ballotitem', 'ballot_ballotreturned', + 'polling_location_pollinglocation', + 'organization_organization', + 'candidate_candidatecampaign', ] dummy_unique_id = 10000000 -LOCAL_TMP_PATH = get_environment_variable('PATH_FOR_TEMP_FILES') or '.' +LOCAL_TMP_PATH = '/tmp/' def retrieve_sql_tables_as_csv(table_name, start, end): """ - Extract one of the 15 allowable database tables to CSV (pipe delimited) and send it to the + Extract one of the (originally) 15 allowable database tables to CSV (pipe delimited) and send it to the developer's local WeVoteServer instance limit is used to specify a number of rows to return (this is the SQL LIMIT clause), non-zero or ignored offset is used to specify the first row to return (this is the SQL OFFSET clause), non-zero or ignored + Note July 2022, re Joe: This call to `https://api.wevoteusa.org/apis/v1/retrieveSQLTables/` has been moved from a + "normal" API server (which was timing out) to a "process" API server with an 1800 second timeout. """ t0 = time.time() status = '' + csv_files = {} try: conn = psycopg2.connect( database=get_environment_variable('DATABASE_NAME'), @@ -80,44 +85,55 @@ def retrieve_sql_tables_as_csv(table_name, start, end): # logger.debug("retrieve_sql_tables_as_csv psycopg2 Connected to DB") - csv_files = {} if table_name in allowable_tables: - cur = conn.cursor() - csv_name = os.path.join(LOCAL_TMP_PATH, table_name + '.csvTemp') - print("exporting to: " + csv_name) - with open(csv_name, 'w') as file: + try: + cur = conn.cursor() + file = StringIO() # Empty file + + # logger.error("experiment: REAL FILE ALLOWED FOR file: " + table_name) if positive_value_exists(end): - sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + end +\ - " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + sql = "COPY (SELECT * FROM public." + table_name + " WHERE id BETWEEN " + start + " AND " + \ + end + " ORDER BY id) TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" else: sql = "COPY " + table_name + " TO STDOUT WITH DELIMITER '|' CSV HEADER NULL '\\N'" + # logger.error("experiment: retrieve_tables sql: " + sql) cur.copy_expert(sql, file, size=8192) - logger.error("retrieve_tables sql: " + sql) - file.close() - with open(csv_name, 'r') as file2: - csv_files[table_name] = file2.read() - file2.close() - os.remove(csv_name) - if "exported" not in status: - status += "exported " - status += table_name + "(" + start + "," + end + "), " - conn.commit() - conn.close() - dt = time.time() - t0 - logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + - ' seconds. start = ' + start + ', end = ' + end) + # logger.error("experiment: after cur.copy_expert ") + file.seek(0) + # logger.error("experiment: retrieve_tables file contents: " + file.readline().strip()) + file.seek(0) + csv_files[table_name] = file.read() + file.close() + # logger.error("experiment: after file close, status " + status) + if "exported" not in status: + status += "exported " + status += table_name + "(" + start + "," + end + "), " + # logger.error("experiment: after status +=, " + status) + # logger.error("experiment: before conn.commit") + conn.commit() + # logger.error("experiment: after conn.commit ") + conn.close() + # logger.error("experiment: after conn.close ") + dt = time.time() - t0 + logger.error('Extracting the "' + table_name + '" table took ' + "{:.3f}".format(dt) + + ' seconds. start = ' + start + ', end = ' + end) + except Exception as e: + logger.error("Real exception in retrieve_sql_tables_as_csv(): " + str(e) + " ") else: status = "the table_name '" + table_name + "' is not in the table list, therefore no table was returned" logger.error(status) + # logger.error("experiment: before results") results = { 'success': True, 'status': status, 'files': csv_files, } + # logger.error("experiment: results returned") return results + # run `pg_dump -f /dev/null wevotedev` on the server to evaluate for a corrupted file except Exception as e: status += "retrieve_tables export_sync_files_to_csv caught " + str(e) logger.error(status) @@ -192,18 +208,30 @@ def retrieve_sql_files_from_master_server(request): save_off_database() for table_name in allowable_tables: - print('Starting on the ' + table_name + ' table, requesting up to 1,000,000 rows') - # if table_name != 'ballot_ballotitem': - # continue + print('Starting on the ' + table_name + ' table, requesting up to 500,000 rows') t1 = time.time() dt = 0 start = 0 - end = 999999 + end = 499999 final_lines_count = 0 while end < 20000000: t2 = time.time() - response = requests.get("https://api.wevoteusa.org/apis/v1/retrieveSQLTables/", - params={'table': table_name, 'start': start, 'end': end}) + request_count = 0 + wait_for_a_http_200 = True + response = {} + + while wait_for_a_http_200: + # To test locally: https://wevotedeveloper.com:8000/apis/v1/retrieveSQLTables/?table=election_election + response = requests.get("https://api.wevoteusa.org/apis/v1/retrieveSQLTables/", + params={'table': table_name, 'start': start, 'end': end}) + request_count += 1 + # print("... https://api.wevoteusa.org/apis/v1/retrieveSQLTables/?table=" + table_name + + # '&start=' + str(start) + '&end=' + str(end)) + # print('... HTTP response #' + str(request_count) + ' for table \'' + table_name + + # '\' start ' + str(start) + ', was an HTTP: ' + str(response.status_code)) + if response.status_code == 200: + wait_for_a_http_200 = False + structured_json = json.loads(response.text) if structured_json['success'] is False: print("FAILED: Did not receive '" + table_name + " from server") @@ -213,12 +241,12 @@ def retrieve_sql_files_from_master_server(request): lines = data.splitlines() if len(lines) == 1: dt = time.time() - t1 - print('... Retrieved ' + str(final_lines_count) + ' lines from the ' + table_name + + print('... Retrieved ' + "{:,}".format(final_lines_count) + ' lines from the ' + table_name + ' table (as JSON) in ' + str(int(dt)) + ' seconds)') break final_lines_count += len(lines) - print('... Intermediate line count from this request of 1M, returned ' + str(len(lines)) + - " rows, cumulative is " + str(final_lines_count)) + print('... Intermediate line count from this request of 500k, returned ' + "{:,}".format(len(lines)) + + " rows, cumulative is " + "{:,}".format(final_lines_count)) if len(lines) > 0: try: @@ -232,7 +260,7 @@ def retrieve_sql_files_from_master_server(request): cur = conn.cursor() - print("... Processing rows " + str(start) + " through " + str(end) + " of table " + table_name + + print("... Processing rows " + "{:,}".format(start) + " through " + "{:,}".format(end) + " of table " + table_name + " data received from master server.") if start == 0: cur.execute("DELETE FROM " + table_name) # Delete all existing data in this table @@ -257,7 +285,7 @@ def retrieve_sql_files_from_master_server(request): dt = time.time() - t1 dt2 = time.time() - t2 dtc = time.time() - t0 - print('... Processing and inserting the chunk of 1M from ' + table_name + ' table took ' + + print('... Processing and inserting the chunk of 500k from ' + table_name + ' table took ' + str(int(dt2)) + ' seconds, cumulative ' + str(int(dtc)) + ' seconds') except Exception as e: @@ -265,8 +293,8 @@ def retrieve_sql_files_from_master_server(request): logger.error(status) finally: - start += 1000000 - end += 1000000 + start += 500000 + end += 500000 # Update the last_value for this table so creating new entries doesn't # throw "django Key (id)= already exists" error diff --git a/share/controllers.py b/share/controllers.py index 2929185e8..6fdc2bfd4 100644 --- a/share/controllers.py +++ b/share/controllers.py @@ -13,6 +13,7 @@ from follow.models import FOLLOWING, FollowOrganizationManager import json from organization.models import OrganizationManager +from position.models import PositionListManager import robot_detection from share.models import SharedItem, SharedLinkClicked, SharedPermissionsGranted from voter.models import VoterDeviceLinkManager, VoterManager @@ -166,9 +167,13 @@ def shared_item_retrieve_for_api( # sharedItemRetrieve measure_we_vote_id = '' office_we_vote_id = '' api_call_coming_from_voter_who_shared = False + shared_by_display_name = '' shared_by_voter_we_vote_id = '' shared_by_organization_type = '' shared_by_organization_we_vote_id = '' + shared_by_we_vote_hosted_profile_image_url_large = '' + shared_by_we_vote_hosted_profile_image_url_medium = '' + shared_by_we_vote_hosted_profile_image_url_tiny = '' shared_item_code_no_opinions = '' shared_item_code_all_opinions = '' site_owner_organization_we_vote_id = '' @@ -197,29 +202,33 @@ def shared_item_retrieve_for_api( # sharedItemRetrieve if not results['shared_item_found']: status += "SHARED_ITEM_NOT_FOUND " results = { - 'status': status, - 'success': False, - 'destination_full_url': destination_full_url, + 'status': status, + 'success': False, + 'candidate_we_vote_id': candidate_we_vote_id, + 'date_first_shared': date_first_shared, + 'destination_full_url': destination_full_url, + 'google_civic_election_id': google_civic_election_id, + 'include_friends_only_positions': include_friends_only_positions, + 'is_ballot_share': is_ballot_share, + 'is_candidate_share': is_candidate_share, + 'is_measure_share': is_measure_share, + 'is_office_share': is_office_share, + 'is_organization_share': is_organization_share, + 'is_ready_share': is_ready_share, + 'measure_we_vote_id': measure_we_vote_id, + 'office_we_vote_id': office_we_vote_id, + 'shared_by_display_name': shared_by_display_name, + 'shared_by_voter_we_vote_id': shared_by_voter_we_vote_id, + 'shared_by_organization_type': shared_by_organization_type, + 'shared_by_organization_we_vote_id': shared_by_organization_we_vote_id, + 'shared_by_we_vote_hosted_profile_image_url_large': shared_by_we_vote_hosted_profile_image_url_large, + 'shared_by_we_vote_hosted_profile_image_url_medium': shared_by_we_vote_hosted_profile_image_url_medium, + 'shared_by_we_vote_hosted_profile_image_url_tiny': shared_by_we_vote_hosted_profile_image_url_tiny, 'shared_item_code_no_opinions': shared_item_code_no_opinions, - 'shared_item_code_all_opinions': shared_item_code_all_opinions, + 'shared_item_code_all_opinions': shared_item_code_all_opinions, + 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, 'url_with_shared_item_code_no_opinions': url_with_shared_item_code_no_opinions, - 'url_with_shared_item_code_all_opinions': url_with_shared_item_code_all_opinions, - 'is_ballot_share': is_ballot_share, - 'is_candidate_share': is_candidate_share, - 'is_measure_share': is_measure_share, - 'is_office_share': is_office_share, - 'is_organization_share': is_organization_share, - 'is_ready_share': is_ready_share, - 'include_friends_only_positions': include_friends_only_positions, - 'google_civic_election_id': google_civic_election_id, - 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, - 'shared_by_voter_we_vote_id': shared_by_voter_we_vote_id, - 'shared_by_organization_type': shared_by_organization_type, - 'shared_by_organization_we_vote_id': shared_by_organization_we_vote_id, - 'candidate_we_vote_id': candidate_we_vote_id, - 'measure_we_vote_id': measure_we_vote_id, - 'office_we_vote_id': office_we_vote_id, - 'date_first_shared': date_first_shared, + 'url_with_shared_item_code_all_opinions': url_with_shared_item_code_all_opinions, } return results @@ -358,26 +367,49 @@ def shared_item_retrieve_for_api( # sharedItemRetrieve # Shared item not clicked pass + position_list = [] + if positive_value_exists(shared_item.shared_by_voter_we_vote_id) \ + and shared_item.shared_item_code_all_opinions == shared_item_code: + position_list_manager = PositionListManager() + results = position_list_manager.retrieve_all_positions_for_voter_simple( + voter_we_vote_id=shared_item.shared_by_voter_we_vote_id) + if results['position_list_found']: + position_list = results['position_list'] + + shared_by_display_name = shared_item.shared_by_display_name \ + if positive_value_exists(shared_item.shared_by_display_name) else '' + shared_by_we_vote_hosted_profile_image_url_large = shared_item.shared_by_we_vote_hosted_profile_image_url_large \ + if positive_value_exists(shared_item.shared_by_we_vote_hosted_profile_image_url_large) else '' + shared_by_we_vote_hosted_profile_image_url_medium = shared_item.shared_by_we_vote_hosted_profile_image_url_medium \ + if positive_value_exists(shared_item.shared_by_we_vote_hosted_profile_image_url_medium) else '' + shared_by_we_vote_hosted_profile_image_url_tiny = shared_item.shared_by_we_vote_hosted_profile_image_url_tiny \ + if positive_value_exists(shared_item.shared_by_we_vote_hosted_profile_image_url_tiny) else '' + results = { - 'status': status, - 'success': success, - 'destination_full_url': shared_item.destination_full_url, - 'is_ballot_share': shared_item.is_ballot_share, - 'is_candidate_share': shared_item.is_candidate_share, - 'is_measure_share': shared_item.is_measure_share, - 'is_office_share': shared_item.is_office_share, - 'is_organization_share': shared_item.is_organization_share, - 'is_ready_share': shared_item.is_ready_share, - 'include_friends_only_positions': include_friends_only_positions, - 'google_civic_election_id': shared_item.google_civic_election_id, - 'shared_by_organization_type': shared_item.shared_by_organization_type, + 'status': status, + 'success': success, + 'destination_full_url': shared_item.destination_full_url, + 'is_ballot_share': shared_item.is_ballot_share, + 'is_candidate_share': shared_item.is_candidate_share, + 'is_measure_share': shared_item.is_measure_share, + 'is_office_share': shared_item.is_office_share, + 'is_organization_share': shared_item.is_organization_share, + 'is_ready_share': shared_item.is_ready_share, + 'include_friends_only_positions': include_friends_only_positions, + 'google_civic_election_id': shared_item.google_civic_election_id, + 'position_list': position_list, + 'shared_by_display_name': shared_by_display_name, + 'shared_by_organization_type': shared_item.shared_by_organization_type, 'shared_by_organization_we_vote_id': shared_item.shared_by_organization_we_vote_id, 'shared_by_voter_we_vote_id': shared_item.shared_by_voter_we_vote_id, + 'shared_by_we_vote_hosted_profile_image_url_large': shared_by_we_vote_hosted_profile_image_url_large, + 'shared_by_we_vote_hosted_profile_image_url_medium': shared_by_we_vote_hosted_profile_image_url_medium, + 'shared_by_we_vote_hosted_profile_image_url_tiny': shared_by_we_vote_hosted_profile_image_url_tiny, 'site_owner_organization_we_vote_id': shared_item.site_owner_organization_we_vote_id, - 'candidate_we_vote_id': shared_item.candidate_we_vote_id, - 'measure_we_vote_id': shared_item.measure_we_vote_id, - 'office_we_vote_id': shared_item.office_we_vote_id, - 'date_first_shared': str(shared_item.date_first_shared), + 'candidate_we_vote_id': shared_item.candidate_we_vote_id, + 'measure_we_vote_id': shared_item.measure_we_vote_id, + 'office_we_vote_id': shared_item.office_we_vote_id, + 'date_first_shared': str(shared_item.date_first_shared), } if api_call_coming_from_voter_who_shared: results['shared_item_code_no_opinions'] = shared_item.shared_item_code_no_opinions @@ -420,9 +452,13 @@ def shared_item_save_for_api( # sharedItemSave hostname = '' measure_we_vote_id = '' office_we_vote_id = '' - shared_by_voter_we_vote_id = '' + shared_by_display_name = None shared_by_organization_type = '' shared_by_organization_we_vote_id = '' + shared_by_voter_we_vote_id = '' + shared_by_we_vote_hosted_profile_image_url_large = None + shared_by_we_vote_hosted_profile_image_url_medium = None + shared_by_we_vote_hosted_profile_image_url_tiny = None shared_item_code_no_opinions = '' shared_item_code_all_opinions = '' site_owner_organization_we_vote_id = '' @@ -469,32 +505,48 @@ def shared_item_save_for_api( # sharedItemSave if not required_variables_for_new_entry or not success: status += "NEW_ORGANIZATION_REQUIRED_VARIABLES_MISSING " results = { - 'status': status, - 'success': False, - 'destination_full_url': destination_full_url, + 'status': status, + 'success': False, + 'candidate_we_vote_id': candidate_we_vote_id, + 'date_first_shared': date_first_shared, + 'destination_full_url': destination_full_url, + 'google_civic_election_id': google_civic_election_id, + 'is_ballot_share': is_ballot_share, + 'is_candidate_share': is_candidate_share, + 'is_measure_share': is_measure_share, + 'is_office_share': is_office_share, + 'is_organization_share': is_organization_share, + 'is_ready_share': is_ready_share, + 'measure_we_vote_id': measure_we_vote_id, + 'office_we_vote_id': office_we_vote_id, + 'shared_by_display_name': shared_by_display_name, + 'shared_by_organization_type': shared_by_organization_type, + 'shared_by_organization_we_vote_id': shared_by_organization_we_vote_id, + 'shared_by_voter_we_vote_id': shared_by_voter_we_vote_id, + 'shared_by_we_vote_hosted_profile_image_url_large': shared_by_we_vote_hosted_profile_image_url_large, + 'shared_by_we_vote_hosted_profile_image_url_medium': shared_by_we_vote_hosted_profile_image_url_medium, + 'shared_by_we_vote_hosted_profile_image_url_tiny': shared_by_we_vote_hosted_profile_image_url_tiny, 'shared_item_code_no_opinions': shared_item_code_no_opinions, - 'shared_item_code_all_opinions': shared_item_code_all_opinions, + 'shared_item_code_all_opinions': shared_item_code_all_opinions, + 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, 'url_with_shared_item_code_no_opinions': url_with_shared_item_code_no_opinions, - 'url_with_shared_item_code_all_opinions': url_with_shared_item_code_all_opinions, - 'is_ballot_share': is_ballot_share, - 'is_candidate_share': is_candidate_share, - 'is_measure_share': is_measure_share, - 'is_office_share': is_office_share, - 'is_organization_share': is_organization_share, - 'is_ready_share': is_ready_share, - 'google_civic_election_id': google_civic_election_id, - 'shared_by_organization_type': shared_by_organization_type, - 'shared_by_organization_we_vote_id': shared_by_organization_we_vote_id, - 'shared_by_voter_we_vote_id': shared_by_voter_we_vote_id, - 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, - 'candidate_we_vote_id': candidate_we_vote_id, - 'measure_we_vote_id': measure_we_vote_id, - 'office_we_vote_id': office_we_vote_id, - 'date_first_shared': date_first_shared, + 'url_with_shared_item_code_all_opinions': url_with_shared_item_code_all_opinions, } return results share_manager = ShareManager() + if positive_value_exists(shared_by_organization_we_vote_id): + results = organization_manager.retrieve_organization_from_we_vote_id( + organization_we_vote_id=shared_by_organization_we_vote_id, + read_only=True) + if results['success'] and results['organization_found']: + shared_by_display_name = results['organization'].organization_name + shared_by_we_vote_hosted_profile_image_url_large = \ + results['organization'].we_vote_hosted_profile_image_url_large + shared_by_we_vote_hosted_profile_image_url_medium = \ + results['organization'].we_vote_hosted_profile_image_url_medium + shared_by_we_vote_hosted_profile_image_url_tiny = \ + results['organization'].we_vote_hosted_profile_image_url_tiny defaults = { 'candidate_we_vote_id': candidate_we_vote_id, 'google_civic_election_id': google_civic_election_id, @@ -506,9 +558,13 @@ def shared_item_save_for_api( # sharedItemSave 'is_ready_share': is_ready_share, 'measure_we_vote_id': measure_we_vote_id, 'office_we_vote_id': office_we_vote_id, + 'shared_by_display_name': shared_by_display_name, 'shared_by_organization_type': shared_by_organization_type, 'shared_by_organization_we_vote_id': shared_by_organization_we_vote_id, 'shared_by_voter_we_vote_id': shared_by_voter_we_vote_id, + 'shared_by_we_vote_hosted_profile_image_url_large': shared_by_we_vote_hosted_profile_image_url_large, + 'shared_by_we_vote_hosted_profile_image_url_medium': shared_by_we_vote_hosted_profile_image_url_medium, + 'shared_by_we_vote_hosted_profile_image_url_tiny': shared_by_we_vote_hosted_profile_image_url_tiny, 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, } create_results = share_manager.update_or_create_shared_item( @@ -526,28 +582,32 @@ def shared_item_save_for_api( # sharedItemSave url_with_shared_item_code_all_opinions = "https://" + hostname + "/-" + shared_item_code_all_opinions results = { - 'status': status, - 'success': success, - 'destination_full_url': destination_full_url, - 'shared_item_code_no_opinions': shared_item_code_no_opinions, - 'shared_item_code_all_opinions': shared_item_code_all_opinions, - 'url_with_shared_item_code_no_opinions': url_with_shared_item_code_no_opinions, - 'url_with_shared_item_code_all_opinions': url_with_shared_item_code_all_opinions, - 'is_ballot_share': is_ballot_share, - 'is_candidate_share': is_candidate_share, - 'is_measure_share': is_measure_share, - 'is_office_share': is_office_share, - 'is_organization_share': is_organization_share, - 'is_ready_share': is_ready_share, - 'google_civic_election_id': google_civic_election_id, - 'shared_by_organization_type': shared_by_organization_type, + 'status': status, + 'success': success, + 'candidate_we_vote_id': candidate_we_vote_id, + 'date_first_shared': date_first_shared, + 'destination_full_url': destination_full_url, + 'google_civic_election_id': google_civic_election_id, + 'is_ballot_share': is_ballot_share, + 'is_candidate_share': is_candidate_share, + 'is_measure_share': is_measure_share, + 'is_office_share': is_office_share, + 'is_organization_share': is_organization_share, + 'is_ready_share': is_ready_share, + 'measure_we_vote_id': measure_we_vote_id, + 'office_we_vote_id': office_we_vote_id, + 'shared_by_display_name': shared_by_display_name, + 'shared_by_organization_type': shared_by_organization_type, 'shared_by_organization_we_vote_id': shared_by_organization_we_vote_id, 'shared_by_voter_we_vote_id': shared_by_voter_we_vote_id, - 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, - 'candidate_we_vote_id': candidate_we_vote_id, - 'measure_we_vote_id': measure_we_vote_id, - 'office_we_vote_id': office_we_vote_id, - 'date_first_shared': date_first_shared, + 'shared_by_we_vote_hosted_profile_image_url_large': shared_by_we_vote_hosted_profile_image_url_large, + 'shared_by_we_vote_hosted_profile_image_url_medium': shared_by_we_vote_hosted_profile_image_url_medium, + 'shared_by_we_vote_hosted_profile_image_url_tiny': shared_by_we_vote_hosted_profile_image_url_tiny, + 'shared_item_code_no_opinions': shared_item_code_no_opinions, + 'shared_item_code_all_opinions': shared_item_code_all_opinions, + 'site_owner_organization_we_vote_id': site_owner_organization_we_vote_id, + 'url_with_shared_item_code_no_opinions': url_with_shared_item_code_no_opinions, + 'url_with_shared_item_code_all_opinions': url_with_shared_item_code_all_opinions, } return results diff --git a/share/models.py b/share/models.py index d0b94fcb3..c391486eb 100644 --- a/share/models.py +++ b/share/models.py @@ -25,11 +25,15 @@ class SharedItem(models.Model): # Code for include_friends_only_positions shared_item_code_all_opinions = models.CharField(max_length=50, null=True, blank=True, unique=True, db_index=True) # The voter and organization id of the person initiating the share + shared_by_display_name = models.TextField(blank=True, null=True) shared_by_voter_we_vote_id = models.CharField(max_length=255, null=True, db_index=True) shared_by_organization_type = models.CharField( verbose_name="type of org", max_length=2, choices=ORGANIZATION_TYPE_CHOICES, default=UNKNOWN) shared_by_organization_we_vote_id = models.CharField(max_length=255, null=True, blank=True, db_index=True) shared_by_state_code = models.CharField(max_length=2, null=True, db_index=True) + shared_by_we_vote_hosted_profile_image_url_large = models.TextField(blank=True, null=True) + shared_by_we_vote_hosted_profile_image_url_medium = models.TextField(blank=True, null=True) + shared_by_we_vote_hosted_profile_image_url_tiny = models.TextField(blank=True, null=True) # The owner of the custom site this share was from site_owner_organization_we_vote_id = models.CharField(max_length=255, null=True, blank=False, db_index=True) google_civic_election_id = models.PositiveIntegerField(default=0, null=True, blank=True) @@ -312,32 +316,59 @@ def update_or_create_shared_item( # TODO: Confirm its not in use shared_item_code_all_opinions = random_string + shared_by_display_name = defaults['shared_by_display_name'] if 'shared_by_display_name' in defaults else None + shared_by_we_vote_hosted_profile_image_url_large = \ + defaults['shared_by_we_vote_hosted_profile_image_url_large'] \ + if 'shared_by_we_vote_hosted_profile_image_url_large' in defaults else None + shared_by_we_vote_hosted_profile_image_url_medium = \ + defaults['shared_by_we_vote_hosted_profile_image_url_medium'] \ + if 'shared_by_we_vote_hosted_profile_image_url_medium' in defaults else None + shared_by_we_vote_hosted_profile_image_url_tiny = \ + defaults['shared_by_we_vote_hosted_profile_image_url_tiny'] \ + if 'shared_by_we_vote_hosted_profile_image_url_tiny' in defaults else None + if shared_item_found: - if positive_value_exists(shared_item_code_no_opinions) \ - or positive_value_exists(shared_item_code_all_opinions) \ - or not positive_value_exists(shared_item.year_as_integer): - # There is a reason to update - try: - change_to_save = False - if positive_value_exists(shared_item_code_no_opinions): - shared_item.shared_item_code_no_opinions = shared_item_code_no_opinions - change_to_save = True - if positive_value_exists(shared_item_code_all_opinions): - shared_item.shared_item_code_all_opinions = shared_item_code_all_opinions - change_to_save = True - if not positive_value_exists(shared_item.year_as_integer): - shared_item.generate_year_as_integer() - change_to_save = True - if change_to_save: - shared_item.save() - shared_item_created = True - success = True - status += "SHARED_ITEM_UPDATED " - except Exception as e: - shared_item_created = False - shared_item = None - success = False - status += "SHARED_ITEM_NOT_UPDATED: " + str(e) + " " + try: + change_to_save = False + if shared_item.shared_by_display_name != shared_by_display_name: + shared_item.shared_by_display_name = shared_by_display_name + change_to_save = True + if shared_item.shared_by_we_vote_hosted_profile_image_url_large \ + != shared_by_we_vote_hosted_profile_image_url_large: + shared_item.shared_by_we_vote_hosted_profile_image_url_large = \ + shared_by_we_vote_hosted_profile_image_url_large + change_to_save = True + if shared_item.shared_by_we_vote_hosted_profile_image_url_medium \ + != shared_by_we_vote_hosted_profile_image_url_medium: + shared_item.shared_by_we_vote_hosted_profile_image_url_medium = \ + shared_by_we_vote_hosted_profile_image_url_medium + change_to_save = True + if shared_item.shared_by_we_vote_hosted_profile_image_url_tiny \ + != shared_by_we_vote_hosted_profile_image_url_tiny: + shared_item.shared_by_we_vote_hosted_profile_image_url_tiny = \ + shared_by_we_vote_hosted_profile_image_url_tiny + change_to_save = True + if positive_value_exists(shared_item_code_no_opinions) and \ + shared_item.shared_item_code_no_opinions != shared_item_code_no_opinions: + shared_item.shared_item_code_no_opinions = shared_item_code_no_opinions + change_to_save = True + if positive_value_exists(shared_item_code_all_opinions) and \ + shared_item.shared_item_code_all_opinions != shared_item_code_all_opinions: + shared_item.shared_item_code_all_opinions = shared_item_code_all_opinions + change_to_save = True + if not positive_value_exists(shared_item.year_as_integer): + shared_item.generate_year_as_integer() + change_to_save = True + if change_to_save: + shared_item.save() + shared_item_created = True + success = True + status += "SHARED_ITEM_UPDATED " + except Exception as e: + shared_item_created = False + shared_item = None + success = False + status += "SHARED_ITEM_NOT_UPDATED: " + str(e) + " " else: try: shared_item = SharedItem.objects.create( @@ -352,9 +383,13 @@ def update_or_create_shared_item( is_ready_share=defaults['is_ready_share'], measure_we_vote_id=defaults['measure_we_vote_id'], office_we_vote_id=defaults['office_we_vote_id'], + shared_by_display_name=shared_by_display_name, shared_by_organization_type=defaults['shared_by_organization_type'], shared_by_organization_we_vote_id=defaults['shared_by_organization_we_vote_id'], shared_by_voter_we_vote_id=shared_by_voter_we_vote_id, + shared_by_we_vote_hosted_profile_image_url_large=shared_by_we_vote_hosted_profile_image_url_large, + shared_by_we_vote_hosted_profile_image_url_medium=shared_by_we_vote_hosted_profile_image_url_medium, + shared_by_we_vote_hosted_profile_image_url_tiny=shared_by_we_vote_hosted_profile_image_url_tiny, shared_item_code_no_opinions=shared_item_code_no_opinions, shared_item_code_all_opinions=shared_item_code_all_opinions, site_owner_organization_we_vote_id=defaults['site_owner_organization_we_vote_id'], diff --git a/sms/controllers.py b/sms/controllers.py index 76bba5b2c..e6063b1bb 100644 --- a/sms/controllers.py +++ b/sms/controllers.py @@ -471,7 +471,7 @@ def schedule_verification_sms(sender_voter_we_vote_id, recipient_voter_we_vote_i return results subject = "Please verify your sms" - original_sender_email_subscription_secret_key = '' + original_sender_sms_subscription_secret_key = '' template_variables_for_json = { "subject": subject, @@ -479,9 +479,12 @@ def schedule_verification_sms(sender_voter_we_vote_id, recipient_voter_we_vote_i "we_vote_url": web_app_root_url_verified, "verify_sms_link": web_app_root_url_verified + "/verify_sms/" + recipient_sms_phone_number_secret_key, - "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + - original_sender_email_subscription_secret_key, - "sms_open_url": WE_VOTE_SERVER_ROOT_URL + "/apis/v1/smsOpen?sms_key=1234", + # "recipient_unsubscribe_url": web_app_root_url_verified + "/settings/notifications/esk/" + + # original_sender_sms_subscription_secret_key, + "recipient_unsubscribe_url": + web_app_root_url_verified + "/unsubscribe/{sms_secret_key}/login".format( + sms_secret_key=original_sender_sms_subscription_secret_key, + ), } template_variables_in_json = json.dumps(template_variables_for_json, ensure_ascii=True) verification_from_sms = "We Vote " # TODO DALE Make system variable diff --git a/support_oppose_deciding/controllers.py b/support_oppose_deciding/controllers.py index 1a6173416..c4a756426 100644 --- a/support_oppose_deciding/controllers.py +++ b/support_oppose_deciding/controllers.py @@ -85,13 +85,13 @@ def positions_count_for_candidate(voter_id, candidate_id, candidate_we_vote_id, # so we make sure we have both of these values to return if positive_value_exists(candidate_id): candidate_manager = CandidateManager() - results = candidate_manager.retrieve_candidate_from_id(candidate_id) + results = candidate_manager.retrieve_candidate_from_id(candidate_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] candidate_we_vote_id = candidate.we_vote_id elif positive_value_exists(candidate_we_vote_id): candidate_manager = CandidateManager() - results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id) + results = candidate_manager.retrieve_candidate_from_we_vote_id(candidate_we_vote_id, read_only=True) if results['candidate_found']: candidate = results['candidate'] candidate_id = candidate.id diff --git a/templates/admin/app-ads.txt b/templates/admin/app-ads.txt new file mode 100644 index 000000000..991fcbd4f --- /dev/null +++ b/templates/admin/app-ads.txt @@ -0,0 +1 @@ +facebook.com, 1097389196952441, DIRECT, c3e20eee3f780d68 \ No newline at end of file diff --git a/templates/admin_tools/data_cleanup_organization_analysis.html b/templates/admin_tools/data_cleanup_organization_analysis.html index 8bb67d38e..057de19b6 100644 --- a/templates/admin_tools/data_cleanup_organization_analysis.html +++ b/templates/admin_tools/data_cleanup_organization_analysis.html @@ -28,8 +28,9 @@

Endorser Analysis: {{ organization.organization_name }}

{{ organization.organization_twitter_handle }} ({{ organization.twitter_followers_count }} Followers) - {% endif %} (Refresh Twitter Details - s)
+ {% endif %} ( + Refresh Twitter Details + )
organization.twitter_user_id: {{ organization.twitter_user_id }}
organization.twitter_name: {{ organization.twitter_name }}
organization.twitter_id_from_link_to_organization: {{ organization.twitter_id_from_link_to_organization }}
@@ -173,7 +174,7 @@

Endorsers with Duplicate (Local) Twitter Info

{{ forloop.counter }} {% if organization.organization_photo_url %} - {% endif %} + {% endif %} {{ organization.organization_name }} {% if organization.organization_twitter_handle %}{{ organization.organization_twitter_handle }}
diff --git a/templates/admin_tools/data_cleanup_organization_list_analysis.html b/templates/admin_tools/data_cleanup_organization_list_analysis.html index 04d6badc5..c920afe2d 100644 --- a/templates/admin_tools/data_cleanup_organization_list_analysis.html +++ b/templates/admin_tools/data_cleanup_organization_list_analysis.html @@ -33,7 +33,7 @@

Endorsers with Unique Twitter Data (Local Data)

{{ forloop.counter }} {% if organization.we_vote_hosted_profile_image_url_medium %} - {% endif %} + {% endif %} {{ organization.organization_name }}
   (See images in new window) @@ -101,7 +101,7 @@

Endorsers with Twitter Collision (Local Data)

{{ forloop.counter }} {% if organization.organization_photo_url %} - {% endif %} + {% endif %} {{ organization.organization_name }} {% if organization.organization_twitter_handle %}{{ organization.organization_twitter_handle }}
@@ -145,7 +145,7 @@

Endorsers with Correctly Linked Twitter Data

{{ forloop.counter }} {% if organization.organization_photo_url %} - {% endif %} + {% endif %} {{ organization.organization_name }} {% if organization.organization_twitter_handle %}{{ organization.organization_twitter_handle }}
diff --git a/templates/admin_tools/data_cleanup_voter_list_analysis.html b/templates/admin_tools/data_cleanup_voter_list_analysis.html index 06eee4fc8..1f2952b4f 100644 --- a/templates/admin_tools/data_cleanup_voter_list_analysis.html +++ b/templates/admin_tools/data_cleanup_voter_list_analysis.html @@ -41,7 +41,7 @@

With twitter_id, twitter_screen_name, facebook_id or email

{{ forloop.counter }} {{ voter.id }} - {% if voter.we_vote_hosted_profile_image_url_medium %}{% endif %} + {% if voter.we_vote_hosted_profile_image_url_medium %}{% endif %} {{ voter.get_full_name|default_if_none:"" }} diff --git a/templates/admin_tools/index.html b/templates/admin_tools/index.html index d0eade21e..7e6cb96b1 100644 --- a/templates/admin_tools/index.html +++ b/templates/admin_tools/index.html @@ -5,6 +5,11 @@ {% block content %} {% load humanize %} +

We Vote Admin Menu

{% if user and not user.is_anonymous %}

@@ -172,8 +177,10 @@

Technical Information

{{ postgres_version }} - Git date: - {{git_merge_date}} + Git commit hash: + + {{git_commit_hash}} +

diff --git a/templates/apis_v1/apis_index.html b/templates/apis_v1/apis_index.html index 968e3231a..4245cdbf0 100644 --- a/templates/apis_v1/apis_index.html +++ b/templates/apis_v1/apis_index.html @@ -293,6 +293,12 @@

Send the voter's contacts for processing. (Voter has explicitly opted in.) + + voterContactSave + + Make changes to specific voterContactEmail entries owned by this voter. + + {### ###} diff --git a/templates/candidate/candidate_edit.html b/templates/candidate/candidate_edit.html index d07725c90..7986af1b2 100644 --- a/templates/candidate/candidate_edit.html +++ b/templates/candidate/candidate_edit.html @@ -526,7 +526,12 @@

- +
diff --git a/templates/candidate/candidate_list.html b/templates/candidate/candidate_list.html index d494d2703..05a1189a6 100644 --- a/templates/candidate/candidate_list.html +++ b/templates/candidate/candidate_list.html @@ -263,9 +263,14 @@

Candidates

target="_blank" >Export Candidates ,    Repair Imported Candidate Names,    {% if find_candidates_linked_to_multiple_offices %} - Show All Candidates + Show All Candidates    {% else %} - Show Candidates Linked to Two Offices + Show Candidates Linked to Two Offices    +{% endif %} +{% if vote_usa_candidates_for_this_state %} + Vote USA Candidates API     +{% else %} + (Vote USA Candidates API requires election and state) {% endif %}
@@ -445,7 +450,7 @@

Candidates

- + diff --git a/templates/election/election_list.html b/templates/election/election_list.html index c044e4ae1..d4db4d958 100644 --- a/templates/election/election_list.html +++ b/templates/election/election_list.html @@ -149,6 +149,9 @@

{% if is_national_election %}{{ national_election.election_name }} — { {{ election.election_day_text }} {% if election.days_until_election > 0 %} (in {{ election.days_until_election }} days){% endif %} + + {% endif %} @@ -211,6 +214,12 @@

{% if is_national_election %}{{ national_election.election_name }} — { start: {{ election.refresh_date_started }} + {% elif election.retrieve_date_completed %} + {{ election.retrieve_date_completed }} + {% elif election.retrieve_date_started %} + + start: {{ election.retrieve_date_started }} + {% endif %} diff --git a/templates/election/election_summary.html b/templates/election/election_summary.html index 69a1bb4f0..1d13e5a41 100644 --- a/templates/election/election_summary.html +++ b/templates/election/election_summary.html @@ -339,7 +339,9 @@

{% if not is_national_election or is_national_election and state_code %}
  • - Ballotpedia: Retrieve Ballots from Bulk Map Points, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + Ballotpedia: Retrieve Ballots from Bulk Map Points, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %}
    @@ -352,7 +354,9 @@

  • - Ballotpedia: Refresh BallotItems from Bulk Map Points, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + Ballotpedia: Refresh BallotItems from Bulk Map Points, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %}
    • @@ -363,7 +367,9 @@

    {% endif %}
  • - Ballotpedia - ALL STATES: Schedule Batch Processes to Retrieve Ballots for this Election (All States), up to 125 at a time + Ballotpedia - ALL STATES: Schedule Batch Processes to Retrieve Ballots for this Election (All States), up to + {{ map_points_retrieved_each_batch_chunk }} + at a time
    • {% endif %}{# if election.use_ballotpedia_as_data_source #} @@ -374,7 +380,9 @@

  • - CTCL: Retrieve Ballots from Bulk Map Points, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + CTCL: Retrieve Ballots from Bulk Map Points, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %}
    @@ -387,7 +395,9 @@

  • - CTCL: Refresh Ballots from Bulk Map Points, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + CTCL: Refresh Ballots from Bulk Map Points, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} (Oldest: {{ ballot_returned_oldest_date }}) @@ -395,7 +405,9 @@

  • - CTCL-IN_DEVELOPMENT: Refresh BallotItems from Voter-entered Street Addresses, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + CTCL-IN_DEVELOPMENT: Refresh BallotItems from Voter-entered Street Addresses, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} (Oldest: {{ ballot_returned_voter_oldest_date }}) @@ -413,7 +425,9 @@

  • - Vote USA: Retrieve Ballots from Bulk Map Points, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + Vote USA: Retrieve Ballots from Bulk Map Points, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %}
    @@ -426,7 +440,9 @@

  • - Vote USA: Refresh Ballots from Bulk Map Points, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + Vote USA: Refresh Ballots from Bulk Map Points, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} (Oldest: {{ ballot_returned_oldest_date }}) @@ -434,7 +450,9 @@

  • - Vote USA: Refresh BallotItems from Voter-entered Street Addresses, up to 125 at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} + Vote USA: Refresh BallotItems from Voter-entered Street Addresses, up to + {{ map_points_retrieved_each_batch_chunk }} + at a time{% if state_code %} (for {{ state_code|upper }}){% endif %} (Oldest: {{ ballot_returned_voter_oldest_date }}) @@ -494,12 +512,16 @@

    {% if election.use_ballotpedia_as_data_source %}
  • - Ballotpedia: Refresh Map Point BallotItems from Ballotpedia from existing BallotReturned entries (up to 125 at a time) (Oldest: {{ ballot_returned_oldest_date }}) + Ballotpedia: Refresh Map Point BallotItems from Ballotpedia from existing BallotReturned entries (up to + {{ map_points_retrieved_each_batch_chunk }} + at a time) (Oldest: {{ ballot_returned_oldest_date }})
  • - Ballotpedia: Refresh Voter BallotItems from Ballotpedia from existing BallotReturned entries (up to 125 at a time) (Oldest: {{ ballot_returned_voter_oldest_date }}) + Ballotpedia: Refresh Voter BallotItems from Ballotpedia from existing BallotReturned entries (up to + {{ map_points_retrieved_each_batch_chunk }} + at a time) (Oldest: {{ ballot_returned_voter_oldest_date }})
    • {% endif %}{# if election.use_ballotpedia_as_data_source #} diff --git a/templates/email_outbound/email_templates/campaignx_friend_has_supported.html b/templates/email_outbound/email_templates/campaignx_friend_has_supported.html index a70bc45c0..a6a6cd46b 100644 --- a/templates/email_outbound/email_templates/campaignx_friend_has_supported.html +++ b/templates/email_outbound/email_templates/campaignx_friend_has_supported.html @@ -1,27 +1,27 @@ {# email_outbound/email_templates/campaignx_friend_has_supported.html #} -{# Later we replace "Your friend" with the sender name. #} +{# Later we replace "Your friend" with the sender name. #} We Vote - +