diff --git a/hashtagsv2/hashtags/factories.py b/hashtagsv2/hashtags/factories.py index db59227..4a35350 100644 --- a/hashtagsv2/hashtags/factories.py +++ b/hashtagsv2/hashtags/factories.py @@ -18,6 +18,9 @@ class Meta: page_title = factory.Faker("word") edit_summary = factory.Faker("sentence") rc_id = random.randint(1, 100000) + # Rows need a revision ID so that they can be checked against the wiki. + # See T277832 and hashtagsv2.hashtags.visibility. + rev_id = factory.Sequence(lambda n: 1000000 + n) has_image = False has_video = False has_audio = False diff --git a/hashtagsv2/hashtags/tests.py b/hashtagsv2/hashtags/tests.py index 9c64f7d..4d728f5 100644 --- a/hashtagsv2/hashtags/tests.py +++ b/hashtagsv2/hashtags/tests.py @@ -2,6 +2,8 @@ from mock import patch from json import loads +import requests + from django.urls import reverse from django.test import TestCase, RequestFactory @@ -9,6 +11,31 @@ from .models import Hashtag from .helpers import split_hashtags from . import views +from . import visibility + + +def all_revisions_visible(domain, rev_ids): + """Stand in for the API, reporting every revision asked about as public.""" + return { + "query": { + "pages": [ + {"revisions": [{"revid": rev_id} for rev_id in rev_ids]}, + ] + } + } + + +def hide_revision(hidden_rev_id): + """Stand in for the API, reporting one revision as hidden by the wiki.""" + + def query_wiki(domain, rev_ids): + response = all_revisions_visible(domain, rev_ids) + for revision in response["query"]["pages"][0]["revisions"]: + if revision["revid"] == hidden_rev_id: + revision["commenthidden"] = True + return response + + return query_wiki class HomepageTest(TestCase): @@ -58,6 +85,14 @@ def setUp(cls): cls.message_patcher = patch("hashtagsv2.hashtags.views.messages.add_message") cls.message_patcher.start() + # Results are checked against the wiki before they are displayed, so + # stop the tests from calling out to the API. See T277832. + cls.visibility_patcher = patch( + "hashtagsv2.hashtags.visibility._query_wiki", + side_effect=all_revisions_visible, + ) + cls.visibility_patcher.start() + @classmethod def setUpClass(cls): super(HashtagSearchTest, cls).setUpClass() @@ -67,6 +102,7 @@ def setUpClass(cls): def tearDown(self): super(HashtagSearchTest, self).tearDown() self.message_patcher.stop() + self.visibility_patcher.stop() def test_split_hashtags1(self): """ @@ -300,6 +336,113 @@ def test_hashtags_download_json_full_query(self): # tests enough. self.assertEqual(len(json_content["Rows"]), 1) + def test_hashtags_download_csv_omits_hidden_rows(self): + """ + A revision that the wiki has hidden is left out of the CSV. + """ + hidden_rev_id = Hashtag.objects.filter(hashtag="hashtag1").first().rev_id + + request = RequestFactory().get(self.download_url, {"query": "hashtag1"}) + with patch( + "hashtagsv2.hashtags.visibility._query_wiki", + side_effect=hide_revision(hidden_rev_id), + ): + response = views.csv_download(request) + + # Header plus the four rows that the wiki still shows. + self.assertEqual(len(response.content.splitlines()), 5) + self.assertNotIn(str(hidden_rev_id).encode(), response.content) + + def test_hashtags_download_json_omits_hidden_rows(self): + """ + A revision that the wiki has hidden is left out of the JSON. + """ + hidden_rev_id = Hashtag.objects.filter(hashtag="hashtag1").first().rev_id + + request = RequestFactory().get(self.download_url, {"query": "hashtag1"}) + with patch( + "hashtagsv2.hashtags.visibility._query_wiki", + side_effect=hide_revision(hidden_rev_id), + ): + response = views.json_download(request) + + json_content = loads(response.content.decode("utf-8")) + + self.assertEqual(len(json_content["Rows"]), 4) + self.assertNotIn( + hidden_rev_id, [row["Revision_ID"] for row in json_content["Rows"]] + ) + + def test_hashtags_download_refuses_large_result_set(self): + """ + A search with more results than we can check does not download. + + We send the user back to the search page instead of a file that is + short for a reason that they cannot see. See T277832. + """ + request = RequestFactory().get(self.download_url, {"query": "hashtag1"}) + + for view in [views.csv_download, views.json_download]: + with ( + patch("hashtagsv2.hashtags.views.EXPORT_VERIFY_LIMIT", 2), + patch("hashtagsv2.hashtags.visibility._query_wiki") as query_wiki, + ): + response = view(request) + + # We refuse before we call the API, so that a large search + # cannot make many calls. + query_wiki.assert_not_called() + self.assertEqual(response.status_code, 302) + self.assertIn("query=hashtag1", response.url) + + def test_hashtags_download_refuses_an_incomplete_check(self): + """ + A download stops if we cannot check every row. + + We must not send a file that is short for a reason that the user + cannot see. See T277832. + """ + request = RequestFactory().get(self.download_url, {"query": "hashtag1"}) + + for view in [views.csv_download, views.json_download]: + with patch("hashtagsv2.hashtags.views.EXPORT_TOTAL_BUDGET_S", -1): + response = view(request) + + self.assertEqual(response.status_code, 302) + self.assertIn("query=hashtag1", response.url) + + def test_hashtags_download_refuses_too_many_api_calls(self): + """ + A download stops if the check needs too many API calls. + + The number of calls depends on how many wikis the results come from, + so the row limit cannot bound it. We refuse before we make the first + call. See T277832. + """ + request = RequestFactory().get(self.download_url, {"query": "hashtag1"}) + + for view in [views.csv_download, views.json_download]: + with ( + patch("hashtagsv2.hashtags.views.EXPORT_MAX_BATCHES", 0), + patch("hashtagsv2.hashtags.visibility._query_wiki") as query_wiki, + ): + response = view(request) + + query_wiki.assert_not_called() + self.assertEqual(response.status_code, 302) + self.assertIn("query=hashtag1", response.url) + + def test_hashtags_downloads_are_not_cached(self): + """ + A cache must not keep a download, because the wiki can hide an edit + after we send it. See T277832. + """ + request = RequestFactory().get(self.download_url, {"query": "hashtag1"}) + + for view in [views.csv_download, views.json_download]: + response = view(request) + self.assertIn("no-store", response["Cache-Control"]) + def test_no_hashtags(self): """ On first setup, the tool has nothing in the database. @@ -343,6 +486,7 @@ def test_and_or_search(self): Test that we get the correct object list when search_type 'and' is provided by user """ + # Two hashtags on one edit, so both rows share the edit's IDs. HashtagFactory( hashtag="hashtag_and_1", timestamp=datetime(2016, 1, 3), @@ -350,6 +494,7 @@ def test_and_or_search(self): page_title="test", edit_summary="test_summary", rc_id=1234, + rev_id=5678, ) HashtagFactory( hashtag="hashtag_and_2", @@ -358,6 +503,7 @@ def test_and_or_search(self): page_title="test", edit_summary="test_summary", rc_id=1234, + rev_id=5678, ) factory = RequestFactory() @@ -432,3 +578,198 @@ def test_audio_filter(self): self.assertEqual(len(object_list), 1) # And it is the correct edit self.assertEqual(object_list[0].rc_id, 1234) + + +class RevisionVisibilityTest(TestCase): + """ + The wiki can hide an edit summary or a username after we record it, so + results are checked before they are displayed. See T277832. + """ + + def setUp(self): + self.url = reverse("index") + self.factory = RequestFactory() + self.message_patcher = patch("hashtagsv2.hashtags.views.messages.add_message") + self.message_patcher.start() + + def tearDown(self): + super(RevisionVisibilityTest, self).tearDown() + self.message_patcher.stop() + + def _results(self, api_response): + """Search for one hashtag, with the API replaced by a fixed answer.""" + with patch( + "hashtagsv2.hashtags.visibility._query_wiki", return_value=api_response + ): + request = self.factory.get(self.url, {"query": "hashtag1"}) + response = views.Index.as_view()(request) + return response.context_data["object_list"] + + def test_hidden_summary_removes_row(self): + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + object_list = self._results( + {"query": {"pages": [{"revisions": [{"revid": 1, "commenthidden": True}]}]}} + ) + self.assertEqual(len(object_list), 0) + + def test_suppressed_revision_removes_row(self): + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + object_list = self._results( + {"query": {"pages": [{"revisions": [{"revid": 1, "suppressed": True}]}]}} + ) + self.assertEqual(len(object_list), 0) + + def test_deleted_page_removes_row(self): + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + object_list = self._results({"query": {"badrevids": {"1": {"revid": 1}}}}) + self.assertEqual(len(object_list), 0) + + def test_hidden_username_removes_row(self): + # We drop the row rather than blank the name, because the user search + # filter still matches on the stored username. + HashtagFactory( + hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org", username="xyz" + ) + object_list = self._results( + {"query": {"pages": [{"revisions": [{"revid": 1, "userhidden": True}]}]}} + ) + self.assertEqual(len(object_list), 0) + + def test_api_error_response_removes_rows(self): + # MediaWiki reports some errors with HTTP 200 and an error object. + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + object_list = self._results({"error": {"code": "badvalue"}}) + self.assertEqual(len(object_list), 0) + + def test_malformed_response_removes_rows(self): + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + object_list = self._results({"query": {"pages": "not a list"}}) + self.assertEqual(len(object_list), 0) + + def test_visible_revision_is_shown(self): + HashtagFactory( + hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org", username="xyz" + ) + object_list = self._results( + {"query": {"pages": [{"revisions": [{"revid": 1}]}]}} + ) + self.assertEqual(len(object_list), 1) + self.assertEqual(object_list[0].username, "xyz") + + def test_row_with_no_revision_id_is_removed(self): + # Log actions such as uploads have no revision ID, so we cannot check + # them and must not show them. + HashtagFactory(hashtag="hashtag1", rev_id=None, domain="en.wikipedia.org") + object_list = self._results({"query": {"pages": []}}) + self.assertEqual(len(object_list), 0) + + def test_api_failure_removes_rows(self): + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + with patch( + "hashtagsv2.hashtags.visibility._query_wiki", + side_effect=requests.RequestException("boom"), + ): + request = self.factory.get(self.url, {"query": "hashtag1"}) + response = views.Index.as_view()(request) + self.assertEqual(len(response.context_data["object_list"]), 0) + + def test_time_budget_stops_further_api_calls(self): + # Results can span many wikis. We must stop before the gunicorn + # worker times out, and not show what we did not check. + HashtagFactory(hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org") + with patch("hashtagsv2.hashtags.visibility.API_TOTAL_BUDGET_S", -1): + with patch("hashtagsv2.hashtags.visibility._query_wiki") as query: + request = self.factory.get(self.url, {"query": "hashtag1"}) + response = views.Index.as_view()(request) + self.assertEqual(len(response.context_data["object_list"]), 0) + query.assert_not_called() + + def test_redact_reports_a_complete_check(self): + """ + redact() says that it got an answer for every row. + """ + rows = [ + HashtagFactory( + hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org" + ).get_values_list() + ] + with patch( + "hashtagsv2.hashtags.visibility._query_wiki", + return_value={"query": {"pages": [{"revisions": [{"revid": 1}]}]}}, + ): + rows_to_show, removed, complete = visibility.redact(rows) + + self.assertEqual(len(rows_to_show), 1) + self.assertEqual(removed, 0) + self.assertTrue(complete) + + def test_redact_reports_a_failed_call(self): + """ + A call that fails makes the check incomplete. + + A download refuses when it sees this, instead of sending a file that + is short for a reason that the user cannot see. + """ + rows = [ + HashtagFactory( + hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org" + ).get_values_list() + ] + with patch( + "hashtagsv2.hashtags.visibility._query_wiki", + side_effect=requests.RequestException("boom"), + ): + rows_to_show, removed, complete = visibility.redact(rows) + + self.assertEqual(rows_to_show, []) + self.assertEqual(removed, 1) + self.assertFalse(complete) + + def test_redact_reports_that_it_ran_out_of_time(self): + rows = [ + HashtagFactory( + hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org" + ).get_values_list() + ] + with patch("hashtagsv2.hashtags.visibility._query_wiki") as query_wiki: + rows_to_show, removed, complete = visibility.redact(rows, budget=-1) + + query_wiki.assert_not_called() + self.assertEqual(rows_to_show, []) + self.assertFalse(complete) + + def test_redact_refuses_too_many_api_calls(self): + """ + redact() counts the calls before it makes the first one. + """ + rows = [ + HashtagFactory( + hashtag="hashtag1", rev_id=1, domain="en.wikipedia.org" + ).get_values_list() + ] + with patch("hashtagsv2.hashtags.visibility._query_wiki") as query_wiki: + rows_to_show, removed, complete = visibility.redact(rows, max_batches=0) + + query_wiki.assert_not_called() + self.assertEqual(rows_to_show, []) + self.assertEqual(removed, 1) + self.assertFalse(complete) + + def test_count_batches_counts_each_wiki_separately(self): + """ + One call takes 50 revisions from one wiki, so each wiki needs its own + call. This is why the row limit cannot bound the number of calls. + """ + self.assertEqual(visibility._count_batches({}), 0) + self.assertEqual( + visibility._count_batches({"en.wikipedia.org": set(range(50))}), 1 + ) + self.assertEqual( + visibility._count_batches({"en.wikipedia.org": set(range(51))}), 2 + ) + self.assertEqual( + visibility._count_batches( + {"en.wikipedia.org": {1}, "commons.wikimedia.org": {2}} + ), + 2, + ) diff --git a/hashtagsv2/hashtags/views.py b/hashtagsv2/hashtags/views.py index 1407f56..1b41178 100644 --- a/hashtagsv2/hashtags/views.py +++ b/hashtagsv2/hashtags/views.py @@ -1,15 +1,25 @@ import csv from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode from django.contrib import messages from django.http import HttpResponse, JsonResponse from django.db.models import Count +from django.shortcuts import redirect +from django.urls import reverse from django.views.generic import ListView, TemplateView +from django.utils.cache import add_never_cache_headers from django.utils.translation import gettext as _ from .forms import SearchForm from .helpers import hashtag_queryset, get_hashtags_context from .models import Hashtag +from .visibility import ( + EXPORT_MAX_BATCHES, + EXPORT_TOTAL_BUDGET_S, + EXPORT_VERIFY_LIMIT, + redact, +) class Index(ListView): @@ -49,6 +59,26 @@ def get_context_data(self, *args, **kwargs): # evaluate the paginated queryset we are displaying. if context["page_obj"]: context = get_hashtags_context(self.request, self.object_list, context) + + # The wiki can hide an edit summary or a username after we record + # it, so check this page of results before we show it. T277832 + # + # We ignore whether the check reached every row. A page holds 20 + # rows, so it needs 20 API calls at most, and the message below + # tells the user that some results are missing. + context["hashtags"], removed, _complete = redact(context["hashtags"]) + context["object_list"] = context["hashtags"] + if removed: + messages.add_message( + self.request, + messages.INFO, + # Translators: Message to be displayed when some results + # were removed because the wiki no longer shows them. + _( + "Some results are not shown. The wiki has hidden them, " + "or we could not confirm that they are still public." + ), + ) elif self.request.GET.get("query"): messages.add_message( self.request, @@ -71,6 +101,15 @@ def get_context_data(self, *args, **kwargs): return context + def get(self, request, *args, **kwargs): + response = super().get(request, *args, **kwargs) + if request.GET.get("query"): + # We checked these results against the wiki as we rendered them. + # A cache must not keep serving them after the wiki hides an + # edit. T277832 + add_never_cache_headers(response) + return response + def get_queryset(self): form = self.form_class(self.request.GET) if form.is_valid(): @@ -94,16 +133,63 @@ def get_queryset(self): return [] +def refuse_download(request, request_dict): + """Send the user back to the search page, and say why.""" + messages.add_message( + request, + messages.INFO, + # Translators: Message to be displayed when we cannot check all of + # the results of a search, so we cannot make a file of them. + _( + "We cannot check all of the results of this search, so we cannot " + "make the file. Make the search smaller, or try again later." + ), + ) + return redirect( + "{path}?{query}".format(path=reverse("index"), query=urlencode(request_dict)) + ) + + +def rows_for_download(request): + """ + Get the rows for a download, after a check against the wikis. + + Returns (rows, refusal). If we cannot check all of the results, `rows` is + None and `refusal` is a response that sends the user back to the search + page. A file that is short for a reason that the user cannot see is worse + than no file. See T277832. + """ + request_dict = request.GET.dict() + hashtags = hashtag_queryset(request_dict) + + # A download sends all of the results, not one page of them. A large + # search needs more API calls than we can make before the request times + # out, so we refuse it before we read the rows. + if hashtags.count() > EXPORT_VERIFY_LIMIT: + return None, refuse_download(request, request_dict) + + rows, _removed, complete = redact( + hashtags, budget=EXPORT_TOTAL_BUDGET_S, max_batches=EXPORT_MAX_BATCHES + ) + + # The check needs too many calls, or it ran out of time, or a call + # failed. We do not know about every row, so we send no file. + if not complete: + return None, refuse_download(request, request_dict) + + return rows, None + + def csv_download(request): # If this fails for large files we should consider # https://docs.djangoproject.com/en/2.1/howto/outputting-csv/#streaming-large-csv-files - request_dict = request.GET.dict() + hashtags, refusal = rows_for_download(request) + if refusal is not None: + return refusal response = HttpResponse(content_type="text/csv") response["Content-Disposition"] = 'attachment; filename="hashtags.csv"' - hashtags = hashtag_queryset(request_dict) - writer = csv.writer(response) writer.writerow( [ @@ -133,13 +219,17 @@ def csv_download(request): ] ) + # We checked these rows against the wikis as we made the file. A cache + # must not send them again after the wiki hides an edit. T277832 + add_never_cache_headers(response) + return response def json_download(request): - request_dict = request.GET.dict() - - hashtags = hashtag_queryset(request_dict) + hashtags, refusal = rows_for_download(request) + if refusal is not None: + return refusal row_list = [] for hashtag in hashtags: @@ -154,7 +244,13 @@ def json_download(request): } ) - return JsonResponse({"Rows": row_list}) + response = JsonResponse({"Rows": row_list}) + + # We checked these rows against the wikis as we made the file. A cache + # must not send them again after the wiki hides an edit. T277832 + add_never_cache_headers(response) + + return response class Docs(TemplateView): diff --git a/hashtagsv2/hashtags/visibility.py b/hashtagsv2/hashtags/visibility.py new file mode 100644 index 0000000..4244d09 --- /dev/null +++ b/hashtagsv2/hashtags/visibility.py @@ -0,0 +1,219 @@ +""" +Check that we may still show the edits we recorded. + +We copy each edit into our database when it happens, and that copy never +changes. The wiki can hide the edit summary or the username later, or delete +the page. We therefore ask the wiki about the results before we show them. +See T277832. + +The check fails closed: if we cannot get an answer, we do not show the row. +It also reports whether it reached every row, so that a caller which must not +send a part of a result, such as a download, can refuse instead. +""" + +import logging +import time +from collections import defaultdict + +import requests + +logger = logging.getLogger(__name__) + +# The API takes 50 values in a multi-value parameter for clients that do not +# have the apihighlimits right. See https://www.mediawiki.org/wiki/API:Query +API_BATCH_SIZE = 50 + +# Connect and read timeouts for one request. +API_TIMEOUT_S = (3.05, 5) + +# The most time we spend on API calls for one page of results. Results can +# span many wikis, and we must answer well inside the gunicorn worker +# timeout. We do not show the rows that we do not reach in time. +API_TOTAL_BUDGET_S = 10.0 + +API_USER_AGENT = "hashtags (https://hashtags.wmcloud.org)" + +# The most rows that we check for one download. A download sends all of the +# results, not one page of them. A larger search needs more API calls than we +# can make before the request times out, so we refuse it. +EXPORT_VERIFY_LIMIT = 5000 + +# The most time that we spend on API calls for one download. This is larger +# than the budget for a page of results, because a download has many more +# rows. It stays well inside the gunicorn worker timeout. +EXPORT_TOTAL_BUDGET_S = 45.0 + +# The most API calls that we make for one download. The number of calls +# depends on how many wikis the results come from, not only on how many rows +# there are, so EXPORT_VERIFY_LIMIT cannot bound the time by itself: 5000 +# rows from 362 wikis need 362 calls. We refuse a download that needs more. +EXPORT_MAX_BATCHES = 150 + +# What we treat as "we could not check this batch". The response comes from +# another service, so we include the errors that a malformed body causes: to +# show a row that we could not read is worse than to drop it. +CHECK_FAILURES = ( + requests.RequestException, + AttributeError, + KeyError, + TypeError, + ValueError, +) + + +def _query_wiki(domain, rev_ids): + """Ask one wiki about up to API_BATCH_SIZE revisions.""" + response = requests.get( + "https://{domain}/w/api.php".format(domain=domain), + params={ + "action": "query", + "prop": "revisions", + "revids": "|".join(str(rev_id) for rev_id in rev_ids), + # We must ask for the user and the comment. The API sends the + # "userhidden" and "commenthidden" markers only for properties + # that we request. We do not keep the values it sends back. + "rvprop": "ids|user|comment|flags", + "format": "json", + "formatversion": "2", + }, + headers={"User-Agent": API_USER_AGENT}, + timeout=API_TIMEOUT_S, + ) + response.raise_for_status() + return response.json() + + +def _read_response(data): + """ + Map an API response to {rev_id: may_show_row}. + + A revision that the wiki does not report is left out, so that the caller + treats it as unsafe to show. + """ + if "error" in data: + # MediaWiki reports some errors with HTTP 200 and an error object. + # Without this we would read an empty result and quietly drop every + # row, which looks the same as "the wiki hid them all". + raise ValueError( + "API error: {code}".format(code=data["error"].get("code", "unknown")) + ) + + may_show = {} + query = data.get("query", {}) + + # The wiki cannot find these. The revision is gone, or its page is + # deleted. Either way the summary is no longer public. + for rev_id in query.get("badrevids", {}): + may_show[int(rev_id)] = False + + for page in query.get("pages", []): + for revision in page.get("revisions", []): + # We drop the whole row rather than redact one field. The hashtag + # itself comes from the edit summary, and a blanked username can + # still be confirmed with the user search filter. + hidden = ( + revision.get("commenthidden", False) + or revision.get("userhidden", False) + or revision.get("suppressed", False) + ) + may_show[int(revision["revid"])] = not hidden + + return may_show + + +def _count_batches(rev_ids_by_domain): + """Count the API calls that a check of these revisions needs.""" + return sum( + (len(rev_ids) + API_BATCH_SIZE - 1) // API_BATCH_SIZE + for rev_ids in rev_ids_by_domain.values() + ) + + +def _check_wikis(rev_ids_by_domain, budget): + """ + Ask each wiki about its revisions, within the time budget. + + Returns (may_show, complete). `may_show` is {(domain, rev_id): + may_show_row}, and anything we did not resolve is left out. `complete` is + False if we did not get an answer for every batch, either because we ran + out of time or because a call failed. + """ + may_show = {} + complete = True + deadline = time.monotonic() + budget + + for domain, rev_ids in rev_ids_by_domain.items(): + rev_ids = sorted(rev_ids) + for start in range(0, len(rev_ids), API_BATCH_SIZE): + if time.monotonic() > deadline: + logger.warning( + "Ran out of time checking revision visibility. " + "The rows we did not reach will not be shown." + ) + return may_show, False + + batch = rev_ids[start : start + API_BATCH_SIZE] + try: + found = _read_response(_query_wiki(domain, batch)) + except CHECK_FAILURES as error: + # Leave the batch unresolved. We drop those rows below, and + # we report the check as incomplete. + logger.warning("Could not check revisions on %s: %s", domain, error) + complete = False + continue + + for rev_id, verdict in found.items(): + may_show[(domain, rev_id)] = verdict + + return may_show, complete + + +def redact(rows, budget=None, max_batches=None): + """ + Remove the rows that the wiki no longer shows publicly. + + `rows` are the named tuples that hashtag_queryset() returns, for one page + of results or for one download. `budget` is the most time in seconds that + we spend on API calls. `max_batches` refuses the whole check before we + make the first call, if the check needs more calls than this. + + Returns (rows_to_show, number_removed, complete). `complete` is False if + we could not check every row. A caller which must not send a part of a + result, such as a download, refuses when it sees this. + """ + # We read the constant here, and not in the signature, so that it stays + # the one place that sets the value. + if budget is None: + budget = API_TOTAL_BUDGET_S + + rows = list(rows) + + rev_ids_by_domain = defaultdict(set) + for row in rows: + if row.rev_id is not None: + rev_ids_by_domain[row.domain].add(row.rev_id) + + # We count the calls before we make the first one, so that we refuse at + # once instead of after we spend the whole budget. + if max_batches is not None and _count_batches(rev_ids_by_domain) > max_batches: + logger.warning( + "A result set needs more than %s API calls to check. " + "We did not check it.", + max_batches, + ) + return [], len(rows), False + + may_show, complete = _check_wikis(rev_ids_by_domain, budget) + + rows_to_show = [] + number_removed = 0 + + for row in rows: + # A row with no revision ID is a log action, such as an upload or a + # page move. We have no key to check it with, so we do not show it. + if may_show.get((row.domain, row.rev_id)): + rows_to_show.append(row) + else: + number_removed += 1 + + return rows_to_show, number_removed, complete diff --git a/requirements/django.txt b/requirements/django.txt index a419a15..5476abf 100644 --- a/requirements/django.txt +++ b/requirements/django.txt @@ -7,3 +7,4 @@ python-dateutil==2.9.0.post0 python-dotenv==1.1.0 django-nose==1.4.6 coverage==4.5.3 +requests==2.34.2