From de59e1ea36906f7f6eb18fcbc4876899de2cb270 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Wed, 23 Apr 2025 15:27:52 +0300 Subject: [PATCH 01/89] ENG-7595 Update MFR Dockerfile and (dev-)requirements.txt files to make it possible to create image and mfr-requirements, mfr containers for py3.13 without errors --- Dockerfile | 10 +++++----- dev-requirements.txt | 4 ++-- requirements.txt | 22 +++++++++++----------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index f73f5c93..52e8b0a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.6-slim-buster +FROM python:3.13-slim # ensure unoconv can locate the uno library ENV PYTHONPATH /usr/lib/python3/dist-packages @@ -43,9 +43,9 @@ RUN usermod -d /home www-data \ RUN mkdir -p /code WORKDIR /code -RUN pip install -U pip==18.1 -RUN pip install setuptools==37.0.0 -RUN pip install unoconv==0.8.2 +RUN pip install -U pip==24.0 +RUN pip install setuptools==69.5.1 +RUN pip install unoconv==0.9.0 COPY ./requirements.txt /code/ @@ -61,4 +61,4 @@ RUN python setup.py develop EXPOSE 7778 -CMD ["gosu", "www-data", "invoke", "server"] +CMD ["gosu", "www-data", "invoke", "server"] \ No newline at end of file diff --git a/dev-requirements.txt b/dev-requirements.txt index 37036c25..c0757f84 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,8 +9,8 @@ coveralls flake8==3.0.4 ipdb mccabe -pydevd==0.0.6 +pydevd==3.3.0 pyflakes pytest==2.8.2 pytest-cov==2.2.0 -pyzmq==14.4.1 +pyzmq==26.1.0 diff --git a/requirements.txt b/requirements.txt index 57563941..a3f9207a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -aiohttp==0.18.4 +aiohttp==3.10.6 chardet==2.3.0 furl==0.4.2 humanfriendly==2.1 -invoke==0.13.0 +invoke==2.2.0 mako==1.0.1 -raven==5.27.0 -setuptools==37.0.0 +sentry-sdk==2.22.0 +setuptools==78.1.0 stevedore==1.2.0 -tornado==4.3 +tornado==6.4.2 # WaterButler -git+https://github.com/CenterForOpenScience/waterbutler.git@0.38.6#egg=waterbutler +git+https://github.com/CenterForOpenScience/waterbutler.git@feature/buff-worms agent==0.1.2 google-auth==1.4.1 @@ -23,7 +23,7 @@ pydocx==0.7.0 # Image olefile==0.44 -Pillow==4.3.0 +Pillow==11.0.0 psd-tools==1.4 # IPython @@ -36,7 +36,7 @@ jinja2==2.10.1 mistune==0.8.1 # Pdf -reportlab==3.6.5 +reportlab==4.2.4 # Pptx # python-pptx==0.5.7 @@ -45,10 +45,10 @@ reportlab==3.6.5 docutils==0.12 # Tabular -pandas==0.25.1 +pandas==2.2.3 xlrd==1.0.0 -h5py==2.7.0 -scipy==0.19.1 +h5py==3.13 +scipy==1.14.1 # Md markdown==2.6.2 From dc995b93677528860d913e3b301f32eb566848cb Mon Sep 17 00:00:00 2001 From: mkovalua Date: Wed, 23 Apr 2025 16:01:26 +0300 Subject: [PATCH 02/89] raven is outdated for py3.13 replace it with sentry_sdk to make it possible to build MFR 3.13 image and create containers afterward --- mfr/server/app.py | 15 +++++++++++++-- mfr/server/handlers/core.py | 9 ++++++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mfr/server/app.py b/mfr/server/app.py index d7f4b2ab..b8b1297e 100644 --- a/mfr/server/app.py +++ b/mfr/server/app.py @@ -7,7 +7,10 @@ import tornado.web import tornado.httpserver import tornado.platform.asyncio -from raven.contrib.tornado import AsyncSentryClient + +import sentry_sdk +from sentry_sdk.integrations.tornado import TornadoIntegration +from sentry_sdk.integrations.logging import LoggingIntegration from mfr import settings from mfr.server import settings as server_settings @@ -46,6 +49,15 @@ def almost_apache_style_log(handler): def make_app(debug): + sentry_logging = LoggingIntegration( + level=logging.INFO, # Capture INFO level and above as breadcrumbs + event_level=None, # Do not send logs of any level as events + ) + sentry_sdk.init( + dsn=settings.SENTRY_DSN, + release=__version__, + integrations=[TornadoIntegration(), sentry_logging, ], + ) app = tornado.web.Application( [ (r'/static/(.*)', tornado.web.StaticFileHandler, {'path': server_settings.STATIC_PATH}), @@ -59,7 +71,6 @@ def make_app(debug): debug=debug, log_function=almost_apache_style_log, ) - app.sentry_client = AsyncSentryClient(settings.SENTRY_DSN, release=__version__) return app diff --git a/mfr/server/handlers/core.py b/mfr/server/handlers/core.py index 4c2bd067..e9afbd00 100644 --- a/mfr/server/handlers/core.py +++ b/mfr/server/handlers/core.py @@ -7,7 +7,7 @@ import tornado.web import tornado.iostream -from raven.contrib.tornado import SentryMixin +import sentry_sdk import waterbutler.core.utils import waterbutler.server.utils @@ -76,7 +76,7 @@ def options(self): self.set_header('Access-Control-Allow-Methods', 'GET, PUT, POST, DELETE'), -class BaseHandler(CorsMixin, tornado.web.RequestHandler, SentryMixin): +class BaseHandler(CorsMixin, tornado.web.RequestHandler): """Base class for the Render and Export handlers. Fetches the file metadata for the file indicated by the ``url`` query parameter and builds the provider caches. Also handles writing output and errors. @@ -159,8 +159,11 @@ async def write_stream(self, stream): return def write_error(self, status_code, exc_info): - self.captureException(exc_info) # Log all non 2XX codes to sentry etype, exc, _ = exc_info + scope = sentry_sdk.get_current_scope() + scope.set_tag('class', etype.__name_) + scope.set_tag('status_code', status_code) + sentry_sdk.capture_exception(exc) # Log all non 2XX codes to sentry if issubclass(etype, exceptions.PluginError): try: # clever errors shouldn't break other things From 943e5f745c94cdb3884b47ef479f56bb93e38fea Mon Sep 17 00:00:00 2001 From: Fitz Elliott Date: Mon, 28 Apr 2025 23:55:26 -0400 Subject: [PATCH 03/89] fix some dockerfile lints --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 52e8b0a0..8fd4097d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.13-slim # ensure unoconv can locate the uno library -ENV PYTHONPATH /usr/lib/python3/dist-packages +ENV PYTHONPATH=/usr/lib/python3/dist-packages RUN usermod -d /home www-data \ && chown www-data:www-data /home \ @@ -55,10 +55,10 @@ RUN pip install --no-cache-dir -r ./requirements.txt COPY ./ /code/ ARG GIT_COMMIT= -ENV GIT_COMMIT ${GIT_COMMIT} +ENV GIT_COMMIT=${GIT_COMMIT} RUN python setup.py develop EXPOSE 7778 -CMD ["gosu", "www-data", "invoke", "server"] \ No newline at end of file +CMD ["gosu", "www-data", "invoke", "server"] From 44b707c1959334d87970c1b8edbfd7aa2c5d4e15 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Wed, 23 Apr 2025 22:23:05 +0300 Subject: [PATCH 04/89] fix Deprecated function, class, or module --- mfr/core/extension.py | 4 ++-- mfr/core/provider.py | 2 +- mfr/server/handlers/core.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mfr/core/extension.py b/mfr/core/extension.py index 928cf222..33e60326 100644 --- a/mfr/core/extension.py +++ b/mfr/core/extension.py @@ -77,7 +77,7 @@ def __init__(self, metadata, file_path, url, assets_url, export_url): def render(self): pass - @abc.abstractproperty + @abc.abstractmethod def file_required(self): """Does the rendering html need the raw file content to display correctly? Syntax-highlighted text files do. Standard image formats do not, since an tag @@ -85,7 +85,7 @@ def file_required(self): """ pass - @abc.abstractproperty + @abc.abstractmethod def cache_result(self): pass diff --git a/mfr/core/provider.py b/mfr/core/provider.py index dac7c0f6..fb600dde 100644 --- a/mfr/core/provider.py +++ b/mfr/core/provider.py @@ -34,7 +34,7 @@ def __init__(self, request, url, action=None): 'url': str(self.url), }) - @abc.abstractproperty + @abc.abstractmethod def NAME(self): raise NotImplementedError diff --git a/mfr/server/handlers/core.py b/mfr/server/handlers/core.py index e9afbd00..ea9f9b94 100644 --- a/mfr/server/handlers/core.py +++ b/mfr/server/handlers/core.py @@ -94,7 +94,7 @@ def __init__(self, *args, **kwargs): self.extension_metrics = MetricsRecord('extension') - @abc.abstractproperty + @abc.abstractmethod def NAME(self): raise NotImplementedError From 24666800b8ceeb27ffb157f36ef59109fddf0b56 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Wed, 23 Apr 2025 22:30:08 +0300 Subject: [PATCH 05/89] fix incorrect call arguments --- mfr/extensions/unoconv/render.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mfr/extensions/unoconv/render.py b/mfr/extensions/unoconv/render.py index a872829d..6914f35f 100644 --- a/mfr/extensions/unoconv/render.py +++ b/mfr/extensions/unoconv/render.py @@ -51,7 +51,8 @@ def render(self): self.metadata.ext, self.file_path, self.export_file_path, - self.map['format'] + self.map['format'], + self.metadata, ) exporter.export() From 9159990cbca9cc77348ff999073ee142b10d9000 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Thu, 24 Apr 2025 11:49:23 +0300 Subject: [PATCH 06/89] fix Deprecated function, class, or module --- mfr/__init__.py | 1 - mfr/core/utils.py | 20 +++++++++----------- mfr/extensions/__init__.py | 1 - mfr/providers/__init__.py | 1 - mfr/server/handlers/core.py | 15 ++++++++++----- mfr/server/handlers/exporters.py | 4 ++-- mfr/server/handlers/renderers.py | 4 ++-- tests/core/test_utils.py | 12 +++++------- tests/documentation/test_entrypoints.py | 4 ++-- tests/server/handlers/test_exporters.py | 4 ++-- tests/server/handlers/test_renderers.py | 4 ++-- 11 files changed, 34 insertions(+), 36 deletions(-) diff --git a/mfr/__init__.py b/mfr/__init__.py index 20230109..5e2406e4 100755 --- a/mfr/__init__.py +++ b/mfr/__init__.py @@ -1,4 +1,3 @@ # This is a namespace package, don't put any functional code in here besides the # declare_namespace call, or it will disappear on install. See: # https://setuptools.readthedocs.io/en/latest/setuptools.html#namespace-packages -__import__('pkg_resources').declare_namespace(__name__) diff --git a/mfr/core/utils.py b/mfr/core/utils.py index 52ae1c2b..5a17c179 100644 --- a/mfr/core/utils.py +++ b/mfr/core/utils.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import entry_points from stevedore import driver from mfr.core import exceptions @@ -110,16 +110,15 @@ def get_renderer_name(name: str) -> str: # `ep_iterator` is an iterable object. Must convert it to a `list` for access. # `list()` can only be called once because the iterator moves to the end after conversion. - ep_iterator = pkg_resources.iter_entry_points(group='mfr.renderers', name=name.lower()) - ep_list = list(ep_iterator) + ep = entry_points().select(group='mfr.renderers', name=name.lower()) # Empty list indicates unsupported file type. Return '' and let `make_renderer()` handle it. - if len(ep_list) == 0: + if len(ep) == 0: return '' # If the file type is supported, there must be only one element in the list. - assert len(ep_list) == 1 - return ep_list[0].attrs[0] + assert len(ep) == 1 + return ep[0].value.split(":")[1].split('.')[0] def get_exporter_name(name: str) -> str: @@ -132,16 +131,15 @@ def get_exporter_name(name: str) -> str: # `ep_iterator` is an iterable object. Must convert it to a `list` for access. # `list()` can only be called once because the iterator moves to the end after conversion. - ep_iterator = pkg_resources.iter_entry_points(group='mfr.exporters', name=name.lower()) - ep_list = list(ep_iterator) + ep = entry_points().select(group='mfr.exporters', name=name.lower()) # Empty list indicates unsupported export type. Return '' and let `make_exporter()` handle it. - if len(ep_list) == 0: + if len(ep) == 0: return '' # If the export type is supported, there must be only one element in the list. - assert len(ep_list) == 1 - return ep_list[0].attrs[0] + assert len(ep) == 1 + return ep[0].value.split(":")[1].split('.')[0] def sizeof_fmt(num, suffix='B'): diff --git a/mfr/extensions/__init__.py b/mfr/extensions/__init__.py index de40ea7c..e69de29b 100644 --- a/mfr/extensions/__init__.py +++ b/mfr/extensions/__init__.py @@ -1 +0,0 @@ -__import__('pkg_resources').declare_namespace(__name__) diff --git a/mfr/providers/__init__.py b/mfr/providers/__init__.py index 5284146e..e69de29b 100644 --- a/mfr/providers/__init__.py +++ b/mfr/providers/__init__.py @@ -1 +0,0 @@ -__import__("pkg_resources").declare_namespace(__name__) diff --git a/mfr/server/handlers/core.py b/mfr/server/handlers/core.py index ea9f9b94..995613fa 100644 --- a/mfr/server/handlers/core.py +++ b/mfr/server/handlers/core.py @@ -3,7 +3,7 @@ import uuid import asyncio import logging -import pkg_resources +from importlib.metadata import entry_points import tornado.web import tornado.iostream @@ -287,10 +287,15 @@ class ExtensionsStaticFileHandler(tornado.web.StaticFileHandler, CorsMixin): def initialize(self): namespace = 'mfr.renderers' module_path = 'mfr.extensions' - self.modules = { - ep.module_name.replace(module_path + '.', ''): os.path.join(ep.dist.location, 'mfr', 'extensions', ep.module_name.replace(module_path + '.', ''), 'static') - for ep in list(pkg_resources.iter_entry_points(namespace)) - } + + self.modules = {} + + for ep in entry_points().select(group=namespace): + module_name = ep.value.split(":")[0] # replacement for ep.module_name + module = module_name.replace(module_path + ".", "").split(".")[0] + dist_location = ep.dist.locate_file("") # replacement for ep.dist.location + static_path = os.path.join(dist_location, 'mfr', 'extensions', module, 'static') + self.modules[module] = static_path async def get(self, module_name, path): try: diff --git a/mfr/server/handlers/exporters.py b/mfr/server/handlers/exporters.py index 1bea4694..13cb2c47 100644 --- a/mfr/server/handlers/exporters.py +++ b/mfr/server/handlers/exporters.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import entry_points import tornado.web @@ -8,7 +8,7 @@ def get(self): """List available exporters""" exporters = {} - for ep in pkg_resources.iter_entry_points(group='mfr.exporters'): + for ep in entry_points().select(group='mfr.exporters'): exporters.update({ep.name: ep.load().__name__}) self.write({ diff --git a/mfr/server/handlers/renderers.py b/mfr/server/handlers/renderers.py index 572e0836..dedbf3d6 100644 --- a/mfr/server/handlers/renderers.py +++ b/mfr/server/handlers/renderers.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import entry_points import tornado.web @@ -8,7 +8,7 @@ def get(self): """List available renderers""" renderers = {} - for ep in pkg_resources.iter_entry_points(group='mfr.renderers'): + for ep in entry_points().select(group='mfr.renderers'): renderers.update({ep.name: ep.load().__name__}) self.write({ diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 7e908e35..7d249ec9 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -1,5 +1,5 @@ import pytest -import pkg_resources +from importlib.metadata import entry_points from mfr.core import utils as mfr_utils @@ -14,9 +14,8 @@ def test_get_renderer_name_explicit_assertions(self): assert mfr_utils.get_renderer_name('.pdf') == 'PdfRenderer' def test_get_renderer_name(self): - entry_points = pkg_resources.iter_entry_points(group='mfr.renderers') - for ep in entry_points: - expected = ep.attrs[0] + for ep in entry_points().select(group='mfr.renderers'): + expected = ep.value.split(":")[1].split('.')[0] assert mfr_utils.get_renderer_name(ep.name) == expected def test_get_renderer_name_no_entry_point(self): @@ -30,9 +29,8 @@ def test_get_exporter_name_explicit_assertions(self): assert mfr_utils.get_exporter_name('.odt') == 'UnoconvExporter' def test_get_exporter_name(self): - entry_points = pkg_resources.iter_entry_points(group='mfr.exporters') - for ep in entry_points: - expected = ep.attrs[0] + for ep in entry_points().select(group='mfr.exporters'): + expected = ep.value.split(":")[1].split('.')[0] assert mfr_utils.get_exporter_name(ep.name) == expected def test_get_exporter_name_no_entry_point(self): diff --git a/tests/documentation/test_entrypoints.py b/tests/documentation/test_entrypoints.py index 88f6bbb3..4512ce42 100644 --- a/tests/documentation/test_entrypoints.py +++ b/tests/documentation/test_entrypoints.py @@ -1,7 +1,7 @@ import os import pytest -import pkg_resources +from importlib.metadata import entry_points class TestEntryPoints: @@ -12,6 +12,6 @@ def test_entry_points(self): readme_path = os.path.join(os.path.dirname((parent_dir)), 'supportedextensions.md') with open(readme_path, 'r') as file: readme_ext = [line.strip()[2:] for line in file if '*' in line] - for ep in pkg_resources.iter_entry_points(group='mfr.renderers'): + for ep in entry_points().select(group='mfr.renderers'): if ep.name != 'none': assert ep.name in readme_ext diff --git a/tests/server/handlers/test_exporters.py b/tests/server/handlers/test_exporters.py index 054b41c5..aee6e27d 100644 --- a/tests/server/handlers/test_exporters.py +++ b/tests/server/handlers/test_exporters.py @@ -1,6 +1,6 @@ import json -import pkg_resources +from importlib.metadata import entry_points from tornado import testing from tests import utils @@ -13,7 +13,7 @@ def test_get_status(self): resp = yield self.http_client.fetch(self.get_url('/exporters')) exporters = {} - for ep in pkg_resources.iter_entry_points(group='mfr.exporters'): + for ep in entry_points().select(group='mfr.exporters'): exporters.update({ep.name: ep.load().__name__}) data = json.loads(resp.body.decode('utf-8')) diff --git a/tests/server/handlers/test_renderers.py b/tests/server/handlers/test_renderers.py index ea0c72a0..3fe598cf 100644 --- a/tests/server/handlers/test_renderers.py +++ b/tests/server/handlers/test_renderers.py @@ -1,6 +1,6 @@ import json -import pkg_resources +from importlib.metadata import entry_points from tornado import testing from tests import utils @@ -13,7 +13,7 @@ def test_get_status(self): resp = yield self.http_client.fetch(self.get_url('/renderers')) renderers = {} - for ep in pkg_resources.iter_entry_points(group='mfr.renderers'): + for ep in entry_points().select(group='mfr.renderers'): renderers.update({ep.name: ep.load().__name__}) data = json.loads(resp.body.decode('utf-8')) From be346fab2da235ab7c65e6e5fa65660aa6135b60 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Thu, 24 Apr 2025 14:47:04 +0300 Subject: [PATCH 07/89] fix Incorrect type --- mfr/server/app.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/mfr/server/app.py b/mfr/server/app.py index b8b1297e..da679ef1 100644 --- a/mfr/server/app.py +++ b/mfr/server/app.py @@ -27,13 +27,23 @@ def sig_handler(sig, frame): - io_loop = tornado.ioloop.IOLoop.instance() + """ + https://stackoverflow.com/questions/34554247/python-tornado-i-o-loop-current-vs-instance-method + https://www.tornadoweb.org/en/branch6.3/_modules/tornado/testing.html + """ + io_loop = tornado.ioloop.IOLoop.current() + loop = io_loop.asyncio_loop # Access the asyncio loop from Tornado def stop_loop(): - if len(asyncio.Task.all_tasks(io_loop)) == 0: - io_loop.stop() - else: + """ + Retrieve all tasks associated with tornado in asyncio loop + Todo: (maybe there is more explicit way to check than 'tornado' in repr(task)) + """ + exists_tornado_task = any(task for task in asyncio.all_tasks(loop) if 'tornado' in repr(task)) + if exists_tornado_task: io_loop.call_later(1, stop_loop) + else: + io_loop.stop() io_loop.add_callback_from_signal(stop_loop) From 672260c999e91684ebbf86897ca0554b354bfad0 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Thu, 24 Apr 2025 15:10:06 +0300 Subject: [PATCH 08/89] fix Unbound local variables --- mfr/providers/osf/provider.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mfr/providers/osf/provider.py b/mfr/providers/osf/provider.py index 4745e8ec..eaa9b001 100644 --- a/mfr/providers/osf/provider.py +++ b/mfr/providers/osf/provider.py @@ -60,6 +60,7 @@ async def metadata(self): """ download_url = await self._fetch_download_url() logger.debug('download_url::{}'.format(download_url)) + metadata = {} if '/file?' in download_url: # URL is for WaterButler v0 API # TODO Remove this when API v0 is officially deprecated From 389c57a0fe56810a85761155e85ac5c432fe7219 Mon Sep 17 00:00:00 2001 From: mkovalua Date: Thu, 24 Apr 2025 16:29:11 +0300 Subject: [PATCH 09/89] use pip install pyupgrade and find . -type f -name '*.py' -exec pyupgrade --py313-plus {} \+ to convert code to python3.13 suitable format --- docs/conf.py | 5 ++--- mfr/core/extension.py | 2 +- mfr/core/metrics.py | 2 +- mfr/core/remote_logging.py | 8 +++---- mfr/core/utils.py | 8 +++---- mfr/extensions/codepygments/render.py | 4 ++-- mfr/extensions/image/export.py | 4 ++-- mfr/extensions/image/render.py | 2 +- mfr/extensions/ipynb/render.py | 4 ++-- mfr/extensions/jamovi/render.py | 12 +++++------ mfr/extensions/jasp/html_processor.py | 1 - mfr/extensions/jasp/render.py | 18 ++++++++-------- mfr/extensions/md/render.py | 2 +- mfr/extensions/pdf/export.py | 4 ++-- mfr/extensions/pdf/render.py | 2 +- mfr/extensions/rst/render.py | 2 +- mfr/extensions/tabular/compat.py | 2 -- mfr/extensions/tabular/libs/stdlib_tools.py | 6 +++--- mfr/extensions/tabular/libs/xlrd_tools.py | 2 +- mfr/extensions/tabular/utilities.py | 4 ++-- mfr/extensions/unoconv/export.py | 2 +- mfr/extensions/utils.py | 4 ++-- mfr/providers/osf/provider.py | 10 ++++----- mfr/server/app.py | 2 +- mfr/server/handlers/core.py | 6 +++--- mfr/server/handlers/export.py | 22 +++++++++---------- mfr/server/handlers/render.py | 14 ++++++------ mfr/settings.py | 8 +++---- tasks.py | 12 +++++------ tests/documentation/test_entrypoints.py | 4 ++-- tests/extensions/audio/test_renderer.py | 2 +- tests/extensions/image/test_exporter.py | 24 ++++++++++----------- tests/extensions/image/test_renderer.py | 12 +++++------ tests/extensions/jasp/test_renderer.py | 1 - tests/extensions/pdf/test_exporter.py | 8 +++---- tests/extensions/pdf/test_renderer.py | 12 +++++------ tests/extensions/tabular/test_renderer.py | 2 +- tests/extensions/tabular/test_xlsx_tools.py | 2 +- tests/extensions/video/test_renderer.py | 2 +- tests/server/handlers/test_cors.py | 4 ++-- 40 files changed, 121 insertions(+), 126 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 42c80454..6d31f10b 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # MFR documentation build configuration file. # @@ -46,8 +45,8 @@ master_doc = 'index' # General information about the project. -project = u'mfr' -copyright = u'2023, Center For Open Science' +project = 'mfr' +copyright = '2023, Center For Open Science' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/mfr/core/extension.py b/mfr/core/extension.py index 33e60326..cec133e7 100644 --- a/mfr/core/extension.py +++ b/mfr/core/extension.py @@ -49,7 +49,7 @@ def __init__(self, metadata, file_path, url, assets_url, export_url): self.metadata = metadata self.file_path = file_path self.url = url - self.assets_url = '{}/{}'.format(assets_url, self._get_module_name()) + self.assets_url = f'{assets_url}/{self._get_module_name()}' self.export_url = export_url self.renderer_metrics = MetricsRecord('renderer') if self._get_module_name(): diff --git a/mfr/core/metrics.py b/mfr/core/metrics.py index bf47eee2..013e09bd 100644 --- a/mfr/core/metrics.py +++ b/mfr/core/metrics.py @@ -138,7 +138,7 @@ def __init__(self, category, name): @property def key(self): """ID string for this subrecord: '{category}_{name}'""" - return '{}_{}'.format(self.category, self.name) + return f'{self.category}_{self.name}' def new_subrecord(self, name): """Creates and saves a new subrecord. The new subrecord will have its category set to the diff --git a/mfr/core/remote_logging.py b/mfr/core/remote_logging.py index bdde5446..8febeeda 100644 --- a/mfr/core/remote_logging.py +++ b/mfr/core/remote_logging.py @@ -104,18 +104,18 @@ async def _send_to_keen(payload, collection, project_id, write_key, action, doma Will raise an excpetion if the event cannot be sent.""" serialized = json.dumps(payload).encode('UTF-8') - logger.debug("Serialized payload: {}".format(serialized)) + logger.debug(f"Serialized payload: {serialized}") headers = { 'Content-Type': 'application/json', 'Authorization': write_key, } - url = '{0}/{1}/projects/{2}/events/{3}'.format(settings.KEEN_API_BASE_URL, + url = '{}/{}/projects/{}/events/{}'.format(settings.KEEN_API_BASE_URL, settings.KEEN_API_VERSION, project_id, collection) async with await aiohttp.request('POST', url, headers=headers, data=serialized) as resp: if resp.status == 201: - logger.info('Successfully logged {} to {} collection in {} Keen'.format(action, collection, domain)) + logger.info(f'Successfully logged {action} to {collection} collection in {domain} Keen') else: raise Exception('Failed to log {} to {} collection in {} Keen. Status: {} Error: {}'.format( action, collection, domain, str(int(resp.status)), await resp.read() @@ -133,7 +133,7 @@ def _scrub_headers_for_keen(payload, MAX_ITERATIONS=10): # if our new scrubbed key is already in the payload, we need to increment it if scrubbed_key in scrubbed_payload: for i in range(1, MAX_ITERATIONS + 1): # try MAX_ITERATION times, then give up & drop it - incremented_key = '{}-{}'.format(scrubbed_key, i) + incremented_key = f'{scrubbed_key}-{i}' if incremented_key not in scrubbed_payload: # we found an unused key! scrubbed_payload[incremented_key] = payload[key] break diff --git a/mfr/core/utils.py b/mfr/core/utils.py index 5a17c179..35b91753 100644 --- a/mfr/core/utils.py +++ b/mfr/core/utils.py @@ -23,7 +23,7 @@ def make_provider(name, request, url, action=None): ).driver except RuntimeError: raise exceptions.MakeProviderError( - '"{}" is not a supported provider'.format(name.lower()), + f'"{name.lower()}" is not a supported provider', namespace='mfr.providers', name=name.lower(), invoke_on_load=True, @@ -144,10 +144,10 @@ def get_exporter_name(name: str) -> str: def sizeof_fmt(num, suffix='B'): if abs(num) < 1000: - return '%3.0f%s' % (num, suffix) + return '{:3.0f}{}'.format(num, suffix) for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1000.0: - return '%3.1f%s%s' % (num, unit, suffix) + return '{:3.1f}{}{}'.format(num, unit, suffix) num /= 1000.0 - return '%.1f%s%s' % (num, 'Y', suffix) + return '{:.1f}{}{}'.format(num, 'Y', suffix) diff --git a/mfr/extensions/codepygments/render.py b/mfr/extensions/codepygments/render.py index a76b30ae..d2873961 100644 --- a/mfr/extensions/codepygments/render.py +++ b/mfr/extensions/codepygments/render.py @@ -80,7 +80,7 @@ def _render_html(self, fp, ext, *args, **kwargs): content = data.decode(encoding) except UnicodeDecodeError as err: raise exceptions.FileDecodingError( - message='Unable to decode file as {}.'.format(encoding), + message=f'Unable to decode file as {encoding}.', extension=ext, category='undecodable', original_exception=err, @@ -89,7 +89,7 @@ def _render_html(self, fp, ext, *args, **kwargs): if content is None: raise exceptions.FileDecodingError( - message='File decoded to undefined using encoding "{}"'.format(encoding), + message=f'File decoded to undefined using encoding "{encoding}"', extension=ext, category='decoded_to_undefined', code=500, diff --git a/mfr/extensions/image/export.py b/mfr/extensions/image/export.py index f95d7b8c..db4ff60f 100644 --- a/mfr/extensions/image/export.py +++ b/mfr/extensions/image/export.py @@ -21,7 +21,7 @@ def export(self): image_type = parts[-1].lower() max_size = {'w': None, 'h': None} if len(parts) == 2: - max_size['w'], max_size['h'] = [int(size) for size in parts[0].split('x')] + max_size['w'], max_size['h'] = (int(size) for size in parts[0].split('x')) self.metrics.merge({ 'type': image_type, 'max_size_w': max_size['w'], @@ -66,7 +66,7 @@ def export(self): image.save(self.output_file_path, image_type) image.close() - except (UnicodeDecodeError, IOError, FileNotFoundError, OSError) as err: + except (UnicodeDecodeError, OSError, FileNotFoundError) as err: os.path.splitext(os.path.split(self.source_file_path)[-1]) raise exceptions.PillowImageError( 'Unable to export the file as a {}, please check that the ' diff --git a/mfr/extensions/image/render.py b/mfr/extensions/image/render.py index 473d11dc..dd846943 100644 --- a/mfr/extensions/image/render.py +++ b/mfr/extensions/image/render.py @@ -24,7 +24,7 @@ def render(self): exported_url = furl.furl(self.export_url) if settings.EXPORT_MAXIMUM_SIZE and settings.EXPORT_TYPE: - exported_url.args['format'] = '{}.{}'.format(settings.EXPORT_MAXIMUM_SIZE, settings.EXPORT_TYPE) + exported_url.args['format'] = f'{settings.EXPORT_MAXIMUM_SIZE}.{settings.EXPORT_TYPE}' elif settings.EXPORT_TYPE: exported_url.args['format'] = settings.EXPORT_TYPE else: diff --git a/mfr/extensions/ipynb/render.py b/mfr/extensions/ipynb/render.py index c93232b3..1ee836ea 100644 --- a/mfr/extensions/ipynb/render.py +++ b/mfr/extensions/ipynb/render.py @@ -24,11 +24,11 @@ def __init__(self, *args, **kwargs): def render(self): try: - with open(self.file_path, 'r') as file_pointer: + with open(self.file_path) as file_pointer: notebook = nbformat.reads(file_pointer.read(), as_version=4) except ValueError as err: raise exceptions.InvalidFormatError( - 'Could not read ipython notebook file. {}'.format(str(err)), + f'Could not read ipython notebook file. {str(err)}', extension=self.metadata.ext, download_url=str(self.metadata.download_url), original_exception=err, diff --git a/mfr/extensions/jamovi/render.py b/mfr/extensions/jamovi/render.py index 8e59a605..d634e89b 100644 --- a/mfr/extensions/jamovi/render.py +++ b/mfr/extensions/jamovi/render.py @@ -30,7 +30,7 @@ def render(self): return self.TEMPLATE.render(base=self.assets_url, body=body) except BadZipFile as err: raise jamovi_exceptions.JamoviRendererError( - '{} {}.'.format(self.MESSAGE_FILE_CORRUPT, str(err)), + f'{self.MESSAGE_FILE_CORRUPT} {str(err)}.', extension=self.metadata.ext, corruption_type='bad_zip', reason=str(err), @@ -76,7 +76,7 @@ def _check_file(self, zip_file): manifest = manifest_data.read().decode('utf-8') except KeyError: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Missing manifest'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Missing manifest', extension=self.metadata.ext, corruption_type='key_error', reason='zip missing manifest', @@ -93,7 +93,7 @@ def _check_file(self, zip_file): break else: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Data-Archive-Version not found.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not found.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='Data-Archive-Version not found.', @@ -104,17 +104,17 @@ def _check_file(self, zip_file): try: if archive_version < self.MINIMUM_VERSION: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Data-Archive-Version is too old.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version is too old.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='Data-Archive-Version not found.', ) except TypeError: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Data-Archive-Version not parsable.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not parsable.', extension=self.metadata.ext, corruption_type='manifest_parse_error', - reason='Data-Archive-Version ({}) not parsable.'.format(version_str), + reason=f'Data-Archive-Version ({version_str}) not parsable.', ) return True diff --git a/mfr/extensions/jasp/html_processor.py b/mfr/extensions/jasp/html_processor.py index 1dbf4547..26289cb2 100644 --- a/mfr/extensions/jasp/html_processor.py +++ b/mfr/extensions/jasp/html_processor.py @@ -1,4 +1,3 @@ - from io import StringIO from html.parser import HTMLParser diff --git a/mfr/extensions/jasp/render.py b/mfr/extensions/jasp/render.py index abcece4b..ae9bd0e6 100644 --- a/mfr/extensions/jasp/render.py +++ b/mfr/extensions/jasp/render.py @@ -29,7 +29,7 @@ def render(self): return self.TEMPLATE.render(base=self.assets_url, body=body) except BadZipFile as err: raise exceptions.JaspFileCorruptError( - '{} Failure to unzip. {}.'.format(self.MESSAGE_FILE_CORRUPT, str(err)), + f'{self.MESSAGE_FILE_CORRUPT} Failure to unzip. {str(err)}.', extension=self.metadata.ext, corruption_type='bad_zip', reason=str(err), @@ -50,7 +50,7 @@ def _render_html(self, zip_file, ext, *args, **kwargs): index = index_data.read().decode('utf-8') except KeyError: raise exceptions.JaspFileCorruptError( - '{} Missing index.html.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Missing index.html.', extension=self.metadata.ext, corruption_type='key_error', reason='zip missing ./index.html', @@ -78,7 +78,7 @@ def _check_file(self, zip_file): manifest, flavor = manifest_data.read().decode('utf-8'), 'java' except KeyError: raise exceptions.JaspFileCorruptError( - '{} Missing manifest'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Missing manifest', extension=self.metadata.ext, corruption_type='key_error', reason='zip missing manifest', @@ -107,7 +107,7 @@ def _verify_java_manifest(self, manifest): createdBy = str(value) if not dataArchiveVersionStr: raise exceptions.JaspFileCorruptError( - '{} Data-Archive-Version not found.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not found.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='Data-Archive-Version not found.', @@ -130,10 +130,10 @@ def _verify_java_manifest(self, manifest): ) except TypeError: raise exceptions.JaspFileCorruptError( - '{} Data-Archive-Version not parsable.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not parsable.', extension=self.metadata.ext, corruption_type='manifest_parse_error', - reason='Data-Archive-Version ({}) not parsable.'.format(dataArchiveVersionStr), + reason=f'Data-Archive-Version ({dataArchiveVersionStr}) not parsable.', ) return @@ -145,7 +145,7 @@ def _verify_json_manifest(self, manifest): jasp_archive_version_str = manifest_data.get('jaspArchiveVersion', None) if not jasp_archive_version_str: raise exceptions.JaspFileCorruptError( - '{} jaspArchiveVersion not found.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} jaspArchiveVersion not found.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='jaspArchiveVersion not found.', @@ -167,10 +167,10 @@ def _verify_json_manifest(self, manifest): ) except TypeError: raise exceptions.JaspFileCorruptError( - '{} jaspArchiveVersion not parsable.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} jaspArchiveVersion not parsable.', extension=self.metadata.ext, corruption_type='manifest_parse_error', - reason='jaspArchiveVersion ({}) not parsable.'.format(jasp_archive_version_str), + reason=f'jaspArchiveVersion ({jasp_archive_version_str}) not parsable.', ) return diff --git a/mfr/extensions/md/render.py b/mfr/extensions/md/render.py index 792f0725..7c75a7c0 100644 --- a/mfr/extensions/md/render.py +++ b/mfr/extensions/md/render.py @@ -27,7 +27,7 @@ def __init__(self, *args, **kwargs): def render(self): """Render a markdown file to html.""" - with open(self.file_path, 'r') as fp: + with open(self.file_path) as fp: body = markdown.markdown(fp.read(), extensions=[EscapeHtml()]) return self.TEMPLATE.render(base=self.assets_url, body=body) diff --git a/mfr/extensions/pdf/export.py b/mfr/extensions/pdf/export.py index 18f7a7f1..06b25223 100644 --- a/mfr/extensions/pdf/export.py +++ b/mfr/extensions/pdf/export.py @@ -66,7 +66,7 @@ def tiff_to_pdf(self, tiff_img, max_size): c.save() def export(self): - logger.debug('pdf-export: format::{}'.format(self.format)) + logger.debug(f'pdf-export: format::{self.format}') parts = self.format.split('.') export_type = parts[-1].lower() max_size = [int(x) for x in parts[0].split('x')] if len(parts) == 2 else None @@ -89,7 +89,7 @@ def export(self): self.tiff_to_pdf(image, max_size) image.close() - except (UnicodeDecodeError, IOError) as err: + except (UnicodeDecodeError, OSError) as err: name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1]) raise exceptions.PillowImageError( 'Unable to export the file as a {}, please check that the ' diff --git a/mfr/extensions/pdf/render.py b/mfr/extensions/pdf/render.py index 147d0cab..c0a10e94 100644 --- a/mfr/extensions/pdf/render.py +++ b/mfr/extensions/pdf/render.py @@ -23,7 +23,7 @@ def render(self): download_url = munge_url_for_localdev(self.metadata.download_url) escaped_name = escape_url_for_template( - '{}{}'.format(self.metadata.name, self.metadata.ext) + f'{self.metadata.name}{self.metadata.ext}' ) logger.debug('extension::{} supported-list::{}'.format(self.metadata.ext, settings.EXPORT_SUPPORTED)) diff --git a/mfr/extensions/rst/render.py b/mfr/extensions/rst/render.py index fd853814..d0d37d8b 100644 --- a/mfr/extensions/rst/render.py +++ b/mfr/extensions/rst/render.py @@ -20,7 +20,7 @@ def __init__(self, *args, **kwargs): self.metrics.add('docutils_version', docutils.__version__) def render(self): - with open(self.file_path, 'r') as fp: + with open(self.file_path) as fp: body = publish_parts(fp.read(), writer_name='html')['html_body'] return self.TEMPLATE.render(base=self.assets_url, body=body) diff --git a/mfr/extensions/tabular/compat.py b/mfr/extensions/tabular/compat.py index e4bca09a..4bb6a198 100644 --- a/mfr/extensions/tabular/compat.py +++ b/mfr/extensions/tabular/compat.py @@ -1,5 +1,3 @@ -from __future__ import division - range = range string_types = (str,) unicode = str diff --git a/mfr/extensions/tabular/libs/stdlib_tools.py b/mfr/extensions/tabular/libs/stdlib_tools.py index 3d361198..ed30f837 100644 --- a/mfr/extensions/tabular/libs/stdlib_tools.py +++ b/mfr/extensions/tabular/libs/stdlib_tools.py @@ -27,7 +27,7 @@ def csv_stdlib(fp): for idx, fieldname in enumerate(reader.fieldnames or []): column_count = sum(1 for column in columns if fieldname == column['name']) if column_count: - unique_fieldname = '{}-{}'.format(fieldname, column_count + 1) + unique_fieldname = f'{fieldname}-{column_count + 1}' reader.fieldnames[idx] = unique_fieldname else: unique_fieldname = fieldname @@ -49,7 +49,7 @@ def csv_stdlib(fp): extension='csv', ) from e else: - raise TabularRendererError('csv.Error: {}'.format(e), extension='csv') from e + raise TabularRendererError(f'csv.Error: {e}', extension='csv') from e if not columns and not rows: raise EmptyTableError('Table empty or corrupt.', extension='csv') @@ -66,7 +66,7 @@ def sav_stdlib(fp): :return: tuple of table headers and data """ csv_file = utilities.sav_to_csv(fp) - with open(csv_file.name, 'r') as file: + with open(csv_file.name) as file: csv_file.close() return csv_stdlib(file) diff --git a/mfr/extensions/tabular/libs/xlrd_tools.py b/mfr/extensions/tabular/libs/xlrd_tools.py index 7afbd145..184c3be4 100644 --- a/mfr/extensions/tabular/libs/xlrd_tools.py +++ b/mfr/extensions/tabular/libs/xlrd_tools.py @@ -31,7 +31,7 @@ def xlsx_xlrd(fp): fields = [ str(value) if not isinstance(value, basestring) and value is not None - else value or 'Unnamed: {0}'.format(index + 1) + else value or f'Unnamed: {index + 1}' for index, value in enumerate(fields) ] diff --git a/mfr/extensions/tabular/utilities.py b/mfr/extensions/tabular/utilities.py index 3996c3bc..79db46c7 100644 --- a/mfr/extensions/tabular/utilities.py +++ b/mfr/extensions/tabular/utilities.py @@ -29,8 +29,8 @@ def data_population(in_data, headers=None): headers = headers or in_data[0] return [ - dict([(header, row[cindex]) - for cindex, header in enumerate(headers)]) + {header: row[cindex] + for cindex, header in enumerate(headers)} for row in in_data ] diff --git a/mfr/extensions/unoconv/export.py b/mfr/extensions/unoconv/export.py index 2bff77d3..1e5c8057 100644 --- a/mfr/extensions/unoconv/export.py +++ b/mfr/extensions/unoconv/export.py @@ -17,7 +17,7 @@ def export(self): run([ UNOCONV_BIN, '-n', - '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(ADDRESS, PORT), + '-c', f'socket,host={ADDRESS},port={PORT};urp;StarOffice.ComponentContext', '-f', self.format, '-o', self.output_file_path, '-vvv', diff --git a/mfr/extensions/utils.py b/mfr/extensions/utils.py index ab43f554..fa3b9ca6 100644 --- a/mfr/extensions/utils.py +++ b/mfr/extensions/utils.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -def munge_url_for_localdev(url: str) -> Tuple: +def munge_url_for_localdev(url: str) -> tuple: """If MFR is being run in a local development environment (i.e. LOCAL_DEVELOPMENT is True), we need to replace the internal host (the one the backend services communicate on, default: 192.168.168.167) with the external host (the one the user provides, default: "localhost") @@ -23,7 +23,7 @@ def munge_url_for_localdev(url: str) -> Tuple: url_obj = url_obj._replace( query=urlencode(query_dict, doseq=True), - netloc='{}:{}'.format(settings.LOCAL_HOST, url_obj.port) + netloc=f'{settings.LOCAL_HOST}:{url_obj.port}' ) return url_obj diff --git a/mfr/providers/osf/provider.py b/mfr/providers/osf/provider.py index eaa9b001..32fe2b48 100644 --- a/mfr/providers/osf/provider.py +++ b/mfr/providers/osf/provider.py @@ -59,7 +59,7 @@ async def metadata(self): differently. """ download_url = await self._fetch_download_url() - logger.debug('download_url::{}'.format(download_url)) + logger.debug(f'download_url::{download_url}') metadata = {} if '/file?' in download_url: # URL is for WaterButler v0 API @@ -83,7 +83,7 @@ async def metadata(self): if response_code != 200: raise exceptions.MetadataError( 'Failed to fetch file metadata from WaterButler. Received response: ', - 'code {} {}'.format(str(response_code), str(response_reason)), + f'code {str(response_code)} {str(response_reason)}', metadata_url=download_url, response=response_reason, provider=self.NAME, @@ -128,7 +128,7 @@ async def metadata(self): unique_key = hashlib.sha256((meta['etag'] + cleaned_url.url).encode('utf-8')).hexdigest() stable_str = '/{}/{}{}'.format(meta['resource'], meta['provider'], meta['path']) stable_id = hashlib.sha256(stable_str.encode('utf-8')).hexdigest() - logger.debug('stable_identifier: str({}) hash({})'.format(stable_str, stable_id)) + logger.debug(f'stable_identifier: str({stable_str}) hash({stable_id})') return provider.ProviderMetadata(name, ext, content_type, unique_key, download_url, stable_id) @@ -140,7 +140,7 @@ async def download(self): if response.status >= 400: resp_text = await response.text() - logger.error('Unable to download file: ({}) {}'.format(response.status, resp_text)) + logger.error(f'Unable to download file: ({response.status}) {resp_text}') raise exceptions.DownloadError( 'Unable to download the requested file, please try again later.', download_url=download_url, @@ -181,7 +181,7 @@ async def _fetch_download_url(self): ) await request.release() - logger.debug('osf-download-resolver: request.status::{}'.format(request.status)) + logger.debug(f'osf-download-resolver: request.status::{request.status}') if request.status != 302: raise exceptions.MetadataError( request.reason, diff --git a/mfr/server/app.py b/mfr/server/app.py index da679ef1..864d1ccf 100644 --- a/mfr/server/app.py +++ b/mfr/server/app.py @@ -104,7 +104,7 @@ def serve(): ssl_options=ssl_options, ) - logger.info("Listening on {0}:{1}".format(server_settings.ADDRESS, server_settings.PORT)) + logger.info(f"Listening on {server_settings.ADDRESS}:{server_settings.PORT}") signal.signal(signal.SIGTERM, partial(sig_handler)) asyncio.get_event_loop().set_debug(server_settings.DEBUG) diff --git a/mfr/server/handlers/core.py b/mfr/server/handlers/core.py index 995613fa..1083a49b 100644 --- a/mfr/server/handlers/core.py +++ b/mfr/server/handlers/core.py @@ -113,7 +113,7 @@ async def prepare(self): provider=settings.PROVIDER_NAME, code=400, ) - logging.debug('target_url::{}'.format(self.url)) + logging.debug(f'target_url::{self.url}') self.provider = utils.make_provider( settings.PROVIDER_NAME, @@ -124,7 +124,7 @@ async def prepare(self): self.metadata = await self.provider.metadata() self.extension_metrics.add('ext', self.metadata.ext) - logging.debug('extension::{}'.format(self.metadata.ext)) + logging.debug(f'extension::{self.metadata.ext}') self.cache_provider = waterbutler.core.utils.make_provider( settings.CACHE_PROVIDER_NAME, @@ -170,7 +170,7 @@ def write_error(self, status_code, exc_info): current, child_type = {}, None for level in reversed(exc.attr_stack): if current: - current = {'{}_{}'.format(level[0], child_type): current} + current = {f'{level[0]}_{child_type}': current} current['child_type'] = child_type current.update(level[1]) current['self_type'] = level[0] diff --git a/mfr/server/handlers/export.py b/mfr/server/handlers/export.py index 080499b8..f3b8576d 100644 --- a/mfr/server/handlers/export.py +++ b/mfr/server/handlers/export.py @@ -31,21 +31,21 @@ async def prepare(self): self.format = format[0].decode('utf-8') self.exporter_name = utils.get_exporter_name(self.metadata.ext) - self.cache_file_id = '{}.{}'.format(self.metadata.unique_key, self.format) + self.cache_file_id = f'{self.metadata.unique_key}.{self.format}' if self.exporter_name: - cache_file_path_str = '/export/{}.{}'.format(self.cache_file_id, self.exporter_name) + cache_file_path_str = f'/export/{self.cache_file_id}.{self.exporter_name}' else: - cache_file_path_str = '/export/{}'.format(self.cache_file_id) + cache_file_path_str = f'/export/{self.cache_file_id}' self.cache_file_path = await self.cache_provider.validate_path(cache_file_path_str) self.source_file_path = await self.local_cache_provider.validate_path( - '/export/{}'.format(self.source_file_id) + f'/export/{self.source_file_id}' ) - self.output_file_id = '{}.{}'.format(self.source_file_path.name, self.format) + self.output_file_id = f'{self.source_file_path.name}.{self.format}' self.output_file_path = await self.local_cache_provider.validate_path( - '/export/{}'.format(self.output_file_id) + f'/export/{self.output_file_id}' ) self.metrics.merge({ 'output_file': { @@ -59,9 +59,9 @@ async def get(self): """Export a file to the format specified via the associated extension library""" # File is already in the requested format - if self.metadata.ext.lower() == ".{}".format(self.format.lower()): + if self.metadata.ext.lower() == f".{self.format.lower()}": await self.write_stream(await self.provider.download()) - logger.info('Exported {} with no conversion.'.format(self.format)) + logger.info(f'Exported {self.format} with no conversion.') self.metrics.add('export.conversion', 'noop') return @@ -69,11 +69,11 @@ async def get(self): try: cached_stream = await self.cache_provider.download(self.cache_file_path) except DownloadError as e: - assert e.code == 404, 'Non-404 DownloadError {!r}'.format(e) - logger.info('No cached file found; Starting export [{}]'.format(self.cache_file_path)) + assert e.code == 404, f'Non-404 DownloadError {e!r}' + logger.info(f'No cached file found; Starting export [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'miss') else: - logger.info('Cached file found; Sending downstream [{}]'.format(self.cache_file_path)) + logger.info(f'Cached file found; Sending downstream [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'hit') self._set_headers() return await self.write_stream(cached_stream) diff --git a/mfr/server/handlers/render.py b/mfr/server/handlers/render.py index a8c75419..842828f7 100644 --- a/mfr/server/handlers/render.py +++ b/mfr/server/handlers/render.py @@ -29,13 +29,13 @@ async def prepare(self): self.cache_file_id = self.metadata.unique_key if self.renderer_name: - cache_file_path_str = '/export/{}.{}'.format(self.cache_file_id, self.renderer_name) + cache_file_path_str = f'/export/{self.cache_file_id}.{self.renderer_name}' else: - cache_file_path_str = '/export/{}'.format(self.cache_file_id) + cache_file_path_str = f'/export/{self.cache_file_id}' self.cache_file_path = await self.cache_provider.validate_path(cache_file_path_str) self.source_file_path = await self.local_cache_provider.validate_path( - '/render/{}'.format(self.source_file_id) + f'/render/{self.source_file_id}' ) async def get(self): @@ -45,7 +45,7 @@ async def get(self): self.metadata, self.source_file_path.full_path, self.url, - '{}://{}/assets'.format(self.request.protocol, self.request.host), + f'{self.request.protocol}://{self.request.host}/assets', self.request.uri.replace('/render?', '/export?', 1) ) @@ -55,11 +55,11 @@ async def get(self): try: cached_stream = await self.cache_provider.download(self.cache_file_path) except waterbutler.core.exceptions.DownloadError as e: - assert e.code == 404, 'Non-404 DownloadError {!r}'.format(e) - logger.info('No cached file found; Starting render [{}]'.format(self.cache_file_path)) + assert e.code == 404, f'Non-404 DownloadError {e!r}' + logger.info(f'No cached file found; Starting render [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'miss') else: - logger.info('Cached file found; Sending downstream [{}]'.format(self.cache_file_path)) + logger.info(f'Cached file found; Sending downstream [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'hit') return await self.write_stream(cached_stream) diff --git a/mfr/settings.py b/mfr/settings.py index 7dd613ad..2527176d 100644 --- a/mfr/settings.py +++ b/mfr/settings.py @@ -80,7 +80,7 @@ def get_object(self, key, default=None): def full_key(self, key): """The name of the envvar which corresponds to this key.""" - return '{}_{}'.format(self.parent, key) if self.parent else key + return f'{self.parent}_{key}' if self.parent else key def child(self, key): """Fetch a sub-dict of the current dict.""" @@ -146,16 +146,16 @@ def child(self, key): try: - config_path = os.environ['{}_CONFIG'.format(PROJECT_NAME.upper())] + config_path = os.environ[f'{PROJECT_NAME.upper()}_CONFIG'] except KeyError: env = os.environ.get('ENV', 'test') - config_path = '{}/{}-{}.json'.format(PROJECT_CONFIG_PATH, PROJECT_NAME, env) + config_path = f'{PROJECT_CONFIG_PATH}/{PROJECT_NAME}-{env}.json' config = SettingsDict() config_path = os.path.expanduser(config_path) if not os.path.exists(config_path): - logging.warning('No \'{}\' configuration file found'.format(config_path)) + logging.warning(f'No \'{config_path}\' configuration file found') else: with open(os.path.expanduser(config_path)) as fp: config = SettingsDict(json.load(fp)) diff --git a/tasks.py b/tasks.py index 8280a6f3..1c8d265c 100644 --- a/tasks.py +++ b/tasks.py @@ -9,7 +9,7 @@ @task def wheelhouse(ctx, develop=False): req_file = 'dev-requirements.txt' if develop else 'requirements.txt' - cmd = 'pip wheel --find-links={} -r {} --wheel-dir={} -c {}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH, CONSTRAINTS_FILE) + cmd = f'pip wheel --find-links={WHEELHOUSE_PATH} -r {req_file} --wheel-dir={WHEELHOUSE_PATH} -c {CONSTRAINTS_FILE}' ctx.run(cmd, pty=True) @@ -17,10 +17,10 @@ def wheelhouse(ctx, develop=False): def install(ctx, develop=False): ctx.run('python setup.py develop') req_file = 'dev-requirements.txt' if develop else 'requirements.txt' - cmd = 'pip install --upgrade -r {} -c {}'.format(req_file, CONSTRAINTS_FILE) + cmd = f'pip install --upgrade -r {req_file} -c {CONSTRAINTS_FILE}' if WHEELHOUSE_PATH: - cmd += ' --no-index --find-links={}'.format(WHEELHOUSE_PATH) + cmd += f' --no-index --find-links={WHEELHOUSE_PATH}' ctx.run(cmd, pty=True) @@ -43,14 +43,14 @@ def test(ctx, verbose=False, nocov=False, extension=None, path=None): # `--extension=` and `--path=` are mutually exclusive options assert not (extension and path) if path: - path = '/{}'.format(path) if path else '' + path = f'/{path}' if path else '' elif extension: - path = '/extensions/{}/'.format(extension) if extension else '' + path = f'/extensions/{extension}/' if extension else '' else: path = '' coverage = ' --cov-report term-missing --cov mfr' if not nocov else '' verbose = '-v' if verbose else '' - cmd = 'py.test{} tests{} {}'.format(coverage, path, verbose) + cmd = f'py.test{coverage} tests{path} {verbose}' ctx.run(cmd, pty=True) diff --git a/tests/documentation/test_entrypoints.py b/tests/documentation/test_entrypoints.py index 4512ce42..c9f02af9 100644 --- a/tests/documentation/test_entrypoints.py +++ b/tests/documentation/test_entrypoints.py @@ -9,8 +9,8 @@ class TestEntryPoints: def test_entry_points(self): parent_dir = os.pardir - readme_path = os.path.join(os.path.dirname((parent_dir)), 'supportedextensions.md') - with open(readme_path, 'r') as file: + readme_path = os.path.join(os.path.dirname(parent_dir), 'supportedextensions.md') + with open(readme_path) as file: readme_ext = [line.strip()[2:] for line in file if '*' in line] for ep in entry_points().select(group='mfr.renderers'): if ep.name != 'none': diff --git a/tests/extensions/audio/test_renderer.py b/tests/extensions/audio/test_renderer.py index 6dc34414..4fa7a465 100644 --- a/tests/extensions/audio/test_renderer.py +++ b/tests/extensions/audio/test_renderer.py @@ -40,7 +40,7 @@ class TestAudioRenderer: def test_render_audio(self, renderer, url): body = renderer.render() assert '