From b22f5a13dbee231205dd505de96bbe640207ac93 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Tue, 21 Apr 2026 12:19:51 -0400 Subject: [PATCH 1/5] ci: move to using pytest and cov ignore tests again --- .github/workflows/tests.yml | 4 ++-- pyproject.toml | 11 +++++++++++ tests/.coveragerc | 14 -------------- 3 files changed, 13 insertions(+), 16 deletions(-) delete mode 100644 tests/.coveragerc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 86276730..a9c976ed 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,8 +39,8 @@ jobs: - name: Run tests run: | - coverage run --parallel-mode run_tests.py prod - DEVELOPMENT=1 coverage run --parallel-mode -m tests.test_web_api + coverage run -m pytest + DEVELOPMENT=1 coverage run tests/test_web_api.py coverage combine coverage xml diff --git a/pyproject.toml b/pyproject.toml index 468fff92..115f3fdd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,6 +129,16 @@ Repository = "https://github.com/ReadAlongs/Studio" Issues = "https://github.com/ReadAlongs/Studio/issues" Changelog = "https://github.com/ReadAlongs/Studio/releases" +[tool.coverage.run] +source_pkgs = ["readalongs"] +branch = true +parallel = true +omit = ["readalongs/waveform2svg/*", "readalongs/epub/*"] +exclude_also = ["if 0:", "if __name__ == .__main__.:"] + +[tool.coverage.report] +precision = 2 + [tool.mypy] plugins = ["pydantic.mypy"] ignore_missing_imports = true @@ -140,4 +150,5 @@ profile = "black" filterwarnings = [ "ignore:'audioop' is deprecated and slated for removal in Python 3.13:DeprecationWarning", "ignore:.*codecs.open.. is deprecated. Use open.. instead.*:DeprecationWarning", + "ignore:.*Module already imported so cannot be rewritten; anyio.*", ] diff --git a/tests/.coveragerc b/tests/.coveragerc deleted file mode 100644 index e4786078..00000000 --- a/tests/.coveragerc +++ /dev/null @@ -1,14 +0,0 @@ -# .coveragerc to control coverage.py -[run] -branch = True -source = readalongs -omit = */readalongs/waveform2svg/* - -[report] -precision = 2 -# Regexes for lines to exclude from consideration -exclude_lines = - # Don't complain if non-runnable code isn't run: - pragma: no cover - if 0: - if __name__ == .__main__.: From d881b1df0ef0c9a29201c715bd0dc0cea5896208 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Tue, 21 Apr 2026 12:21:56 -0400 Subject: [PATCH 2/5] refactor(tests): use pytest.main() for better output in all suites --- tests/test_align_cli.py | 6 +++--- tests/test_anchors.py | 6 ++++-- tests/test_api.py | 5 +++-- tests/test_audio.py | 6 ++++-- tests/test_config.py | 6 ++++-- tests/test_dna_text.py | 5 +++-- tests/test_dna_utils.py | 7 +++++-- tests/test_dtd.py | 6 ++++-- tests/test_force_align.py | 5 +++-- tests/test_g2p_cli.py | 5 +++-- tests/test_make_xml_cli.py | 6 ++++-- tests/test_misc.py | 5 +++-- tests/test_package_urls.py | 5 +++-- tests/test_silence.py | 5 +++-- tests/test_smil.py | 6 ++++-- tests/test_temp_file.py | 9 ++++++--- tests/test_tokenize_cli.py | 6 ++++-- tests/test_tokenize_xml.py | 6 ++++-- tests/test_web_api.py | 6 ++++-- 19 files changed, 71 insertions(+), 40 deletions(-) diff --git a/tests/test_align_cli.py b/tests/test_align_cli.py index 168e2bd7..317b02c0 100755 --- a/tests/test_align_cli.py +++ b/tests/test_align_cli.py @@ -6,13 +6,14 @@ import os import subprocess +import sys import tempfile from os.path import exists, join from pathlib import Path from typing import Union -from unittest import main from lxml.html import fromstring +from pytest import main from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION from readalongs.cli import align, langs @@ -152,7 +153,6 @@ def test_invoke_align(self) -> None: str(output), ], ) - print("dir(result)", dir(results_output_exists)) self.assertNotEqual(results_output_exists.exit_code, 0) self.assertIn( "already exists, use -f to overwrite", results_output_exists.output @@ -651,4 +651,4 @@ def test_ffmpeg_is_present(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_anchors.py b/tests/test_anchors.py index df2a21f5..04a099d2 100755 --- a/tests/test_anchors.py +++ b/tests/test_anchors.py @@ -3,9 +3,11 @@ """Unit testing for the anchors functionality in readalongs align""" import os +import sys from contextlib import redirect_stderr from io import StringIO -from unittest import main + +from pytest import main from readalongs.align import align_audio from readalongs.log import LOGGER @@ -101,4 +103,4 @@ def test_anchors_align_modes(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_api.py b/tests/test_api.py index 52dc96a3..38d12337 100755 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -5,11 +5,12 @@ """ import re +import sys from contextlib import redirect_stderr from io import StringIO -from unittest import main import click +from pytest import main from readalongs import api from readalongs.log import LOGGER @@ -219,4 +220,4 @@ def test_extract_version_from_url(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_audio.py b/tests/test_audio.py index 307387a2..51436d5c 100755 --- a/tests/test_audio.py +++ b/tests/test_audio.py @@ -3,9 +3,11 @@ """Test suite for various audio contents handling methods""" import os +import sys from pathlib import Path from subprocess import run -from unittest import main + +from pytest import main from readalongs.audio_utils import ( extract_section, @@ -165,4 +167,4 @@ def test_write_audio_to_file(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_config.py b/tests/test_config.py index 3647c964..96ae1296 100755 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -4,10 +4,12 @@ import io import os +import sys from contextlib import redirect_stderr -from unittest import TestCase, main +from unittest import TestCase from lxml import etree +from pytest import main from readalongs.text.add_elements_to_xml import add_images, add_supplementary_xml from readalongs.text.util import load_xml @@ -79,4 +81,4 @@ def test_arbitrary_xml(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_dna_text.py b/tests/test_dna_text.py index 93253900..2d9cf317 100755 --- a/tests/test_dna_text.py +++ b/tests/test_dna_text.py @@ -2,11 +2,12 @@ """Test handling of DNA text in tokenization""" +import sys from contextlib import redirect_stderr from io import StringIO -from unittest import main from lxml import etree +from pytest import main from readalongs.text import tokenize_xml from readalongs.text.add_ids_to_xml import add_ids @@ -163,4 +164,4 @@ def test_dna_word_nested(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_dna_utils.py b/tests/test_dna_utils.py index fb21e902..cf7b0153 100755 --- a/tests/test_dna_utils.py +++ b/tests/test_dna_utils.py @@ -2,7 +2,10 @@ """Test suite for DNA segment manupulation methods""" -from unittest import TestCase, main +import sys +from unittest import TestCase + +from pytest import main from readalongs.dna_utils import ( calculate_adjustment, @@ -175,4 +178,4 @@ def test_dna_union(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_dtd.py b/tests/test_dtd.py index 3c216a1c..45d199b2 100644 --- a/tests/test_dtd.py +++ b/tests/test_dtd.py @@ -3,10 +3,12 @@ """Test our XML DTD to make sure all valid examples validate and invalid ones don't""" import os +import sys from os.path import dirname -from unittest import TestCase, main +from unittest import TestCase from lxml import etree +from pytest import main from readalongs.text.util import load_xml @@ -110,4 +112,4 @@ def test_backwards_compatibility(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_force_align.py b/tests/test_force_align.py index 1c9bd239..471825e2 100755 --- a/tests/test_force_align.py +++ b/tests/test_force_align.py @@ -6,13 +6,14 @@ import os import shutil -import unittest +import sys import wave from contextlib import redirect_stderr from io import StringIO from tempfile import TemporaryDirectory from lxml import etree +from pytest import main from soundswallower import get_model_path from readalongs.align import align_audio @@ -213,4 +214,4 @@ def test_convert_no_version(self): if __name__ == "__main__": LOGGER.setLevel("DEBUG") - unittest.main() + main([__file__, *sys.argv]) diff --git a/tests/test_g2p_cli.py b/tests/test_g2p_cli.py index 15261034..b5063de2 100755 --- a/tests/test_g2p_cli.py +++ b/tests/test_g2p_cli.py @@ -4,11 +4,12 @@ import os import re +import sys from contextlib import redirect_stderr from io import StringIO -from unittest import main from lxml import etree +from pytest import main from readalongs.align import align_audio from readalongs.cli import align, g2p, make_xml, tokenize @@ -489,4 +490,4 @@ def test_non_convertible_words(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_make_xml_cli.py b/tests/test_make_xml_cli.py index 0c0b36d5..4f0b93de 100755 --- a/tests/test_make_xml_cli.py +++ b/tests/test_make_xml_cli.py @@ -5,8 +5,10 @@ import io import os import re +import sys from shutil import copyfile -from unittest import main + +from pytest import main # from readalongs.log import LOGGER from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION @@ -299,4 +301,4 @@ def text2lines(text: str): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_misc.py b/tests/test_misc.py index 943d520a..a9ec95ed 100755 --- a/tests/test_misc.py +++ b/tests/test_misc.py @@ -4,11 +4,12 @@ import itertools import os -from unittest import main +import sys import click from lxml import etree from pep440 import is_canonical +from pytest import main from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION from readalongs.align import split_silences @@ -297,4 +298,4 @@ def test_version_is_pep440_compliant(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_package_urls.py b/tests/test_package_urls.py index 6247bd14..ad282aa7 100755 --- a/tests/test_package_urls.py +++ b/tests/test_package_urls.py @@ -1,8 +1,9 @@ #!/usr/bin/env python -from unittest import main +import sys import requests +from pytest import main from readalongs.text.make_package import ( FONTS_BUNDLE_URL, @@ -46,4 +47,4 @@ def test_fetch_bundles_fallback(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_silence.py b/tests/test_silence.py index a1bcff95..84dd4362 100755 --- a/tests/test_silence.py +++ b/tests/test_silence.py @@ -3,9 +3,10 @@ """Test suite for inserting silences into a readalong""" import os -from unittest import main +import sys from pydub import AudioSegment +from pytest import main from readalongs.cli import align from readalongs.text.util import load_xml @@ -83,4 +84,4 @@ def test_bad_silence(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_smil.py b/tests/test_smil.py index 8bd08fa4..ffd2b9fa 100644 --- a/tests/test_smil.py +++ b/tests/test_smil.py @@ -4,8 +4,10 @@ Unit test suite for the smil writing and parsing utilities """ +import sys from textwrap import dedent -from unittest import main + +from pytest import main from readalongs.text.make_smil import make_smil, parse_smil from tests.basic_test_case import BasicTestCase @@ -104,4 +106,4 @@ def test_parse_bad_smil(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_temp_file.py b/tests/test_temp_file.py index 8ec0adaf..768041c3 100755 --- a/tests/test_temp_file.py +++ b/tests/test_temp_file.py @@ -3,14 +3,17 @@ """Test PortableNamedTemporaryFile class""" import os -import unittest +import sys from tempfile import NamedTemporaryFile +from unittest import TestCase + +from pytest import main from readalongs.log import LOGGER from readalongs.portable_tempfile import PortableNamedTemporaryFile -class TestTempFile(unittest.TestCase): +class TestTempFile(TestCase): """Test PortableNamedTemporaryFile class""" def test_basic_file(self): @@ -98,4 +101,4 @@ def test_seek(self): if __name__ == "__main__": LOGGER.setLevel("DEBUG") - unittest.main() + main([__file__, *sys.argv]) diff --git a/tests/test_tokenize_cli.py b/tests/test_tokenize_cli.py index e915ac6a..e730e229 100755 --- a/tests/test_tokenize_cli.py +++ b/tests/test_tokenize_cli.py @@ -3,7 +3,9 @@ """Test suite for readalongs tokenize""" import os -from unittest import main +import sys + +from pytest import main from readalongs.cli import make_xml, tokenize from tests.basic_test_case import BasicTestCase @@ -66,4 +68,4 @@ def test_bad_input(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_tokenize_xml.py b/tests/test_tokenize_xml.py index 782488c4..ad60a803 100755 --- a/tests/test_tokenize_xml.py +++ b/tests/test_tokenize_xml.py @@ -2,11 +2,13 @@ """Unit test suite for our XML tokenizer module""" +import sys from contextlib import redirect_stderr from io import StringIO -from unittest import TestCase, main +from unittest import TestCase from lxml import etree +from pytest import main from readalongs.text import tokenize_xml from readalongs.text.util import parse_xml @@ -85,4 +87,4 @@ def test_comments(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) diff --git a/tests/test_web_api.py b/tests/test_web_api.py index ec9327cd..3c0bb791 100755 --- a/tests/test_web_api.py +++ b/tests/test_web_api.py @@ -3,13 +3,15 @@ import json import os import re +import sys from contextlib import redirect_stderr from io import StringIO from textwrap import dedent from time import perf_counter -from unittest import main from unittest.mock import patch +from pytest import main + from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION from readalongs.log import LOGGER from readalongs.text.add_ids_to_xml import add_ids @@ -593,4 +595,4 @@ def test_convert_to_bad_format(self): if __name__ == "__main__": - main() + main([__file__, *sys.argv]) From adf73c95512848d8f30e5dba08a0a3b6004722b0 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Tue, 21 Apr 2026 17:04:25 -0400 Subject: [PATCH 3/5] refactor(tests): make run_tests.py use pytest for all, prod and dev --- run_tests.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/run_tests.py b/run_tests.py index 37b021a9..59ce93e2 100755 --- a/run_tests.py +++ b/run_tests.py @@ -19,6 +19,8 @@ import sys from unittest import TestLoader, TestSuite, TextTestRunner +import pytest + from readalongs.log import LOGGER from tests.test_align_cli import TestAlignCli from tests.test_anchors import TestAnchors @@ -111,6 +113,11 @@ def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool: LOGGER.info("No test suite specified, defaulting to dev.") suite = "dev" + if not describe and suite in ("prod", "all", "dev"): + pytest_args = [] if verbosity < 2 else ["--verbose"] + rc = pytest.main(pytest_args) + return rc == 0 + if suite == "e2e": test_suite = TestSuite(e2e_tests) elif suite == "api": From 57fc1aa68a43e0aa8c0da020ab193a7b16a88178 Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Tue, 21 Apr 2026 17:35:04 -0400 Subject: [PATCH 4/5] refactor(tests): remove unused cruft in run_tests and make it use pytest --- run_tests.py | 130 +++++++++------------------------------------------ 1 file changed, 22 insertions(+), 108 deletions(-) diff --git a/run_tests.py b/run_tests.py index 59ce93e2..ff9259b1 100755 --- a/run_tests.py +++ b/run_tests.py @@ -4,107 +4,37 @@ Top-level runner for out test suites Invoke as - ./run.py [suite] + ./run_tests.py [suite] where [suite] can be one of: all: run everything, by searching the directory for all test suite files prod: synonym for all - dev: run the standard development test suite - this is what we do in CI + dev: now also a synonym for all e2e: run the end-to-end tests - other: run the other tests + api: run the API-related tests """ import argparse -import os -import re import sys -from unittest import TestLoader, TestSuite, TextTestRunner +from pathlib import Path import pytest from readalongs.log import LOGGER -from tests.test_align_cli import TestAlignCli -from tests.test_anchors import TestAnchors -from tests.test_api import TestAlignApi -from tests.test_audio import TestAudio -from tests.test_config import TestConfig -from tests.test_dna_text import TestDNAText -from tests.test_dna_utils import TestDNAUtils -from tests.test_dtd import TestDTD -from tests.test_force_align import TestForceAlignment, TestXHTML -from tests.test_g2p_cli import TestG2pCli -from tests.test_make_xml_cli import TestMakeXMLCli -from tests.test_misc import TestMisc -from tests.test_package_urls import TestPackageURLs -from tests.test_silence import TestSilence -from tests.test_smil import TestSmilUtilities -from tests.test_temp_file import TestTempFile -from tests.test_tokenize_cli import TestTokenizeCli -from tests.test_tokenize_xml import TestTokenizer -from tests.test_web_api import TestWebApi - -LOADER = TestLoader() - -e2e_tests = [ - LOADER.loadTestsFromTestCase(test) for test in (TestForceAlignment, TestXHTML) -] - -api_tests = [ - LOADER.loadTestsFromTestCase(test) for test in [TestWebApi] -] # TODO: add some load testing with https://locust.io/ - -other_tests = [ - LOADER.loadTestsFromTestCase(test) - for test in [ - TestAnchors, - TestConfig, - TestDNAText, - TestDNAUtils, - TestTokenizer, - TestTokenizeCli, - TestTempFile, - TestMakeXMLCli, - TestAudio, - TestAlignCli, - TestAlignApi, - TestG2pCli, - TestMisc, - TestSilence, - TestSmilUtilities, - TestPackageURLs, - TestWebApi, - TestDTD, - ] -] - - -def list_tests(suite: TestSuite): - for subsuite in suite: - for match in re.finditer(r"tests=\[([^][]+)\]>", str(subsuite)): - yield from match[1].split(", ") - - -def describe_suite(suite: TestSuite): - full_suite = LOADER.discover(os.path.dirname(__file__)) - full_list = list(list_tests(full_suite)) - requested_list = list(list_tests(suite)) - requested_set = set(requested_list) - print("Test suite includes:", *sorted(requested_list), sep="\n") - print( - "\nTest suite excludes:", - *sorted(test for test in full_list if test not in requested_set), - sep="\n", - ) +e2e_tests = ["test_force_align", "test_align_cli"] + +# TODO: add some load testing with https://locust.io/ +api_tests = ["test_web_api", "test_api"] -SUITES = ["all", "dev", "e2e", "prod", "api", "other"] +SUITES = ["all", "dev", "e2e", "prod", "api"] -def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool: + +def run_tests(suite: str, verbose: bool = True) -> bool: """Run the specified test suite. Args: suite: one of SUITES, "dev" if the empty string - describe: if True, list all the test cases instead of running them. Returns: True iff success """ @@ -113,44 +43,28 @@ def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool: LOGGER.info("No test suite specified, defaulting to dev.") suite = "dev" - if not describe and suite in ("prod", "all", "dev"): - pytest_args = [] if verbosity < 2 else ["--verbose"] - rc = pytest.main(pytest_args) - return rc == 0 - if suite == "e2e": - test_suite = TestSuite(e2e_tests) + test_suite = e2e_tests elif suite == "api": - test_suite = TestSuite(api_tests) - elif suite == "dev": - test_suite = TestSuite(other_tests + e2e_tests) - elif suite in ("prod", "all"): - test_suite = LOADER.discover(os.path.dirname(__file__)) - elif suite == "other": - test_suite = TestSuite(other_tests) + test_suite = api_tests + elif suite in ("prod", "all", "dev"): + test_suite = [] else: LOGGER.error( "Sorry, you need to select a Test Suite to run, one of: " + " ".join(SUITES) ) return False - if describe: - describe_suite(test_suite) - return True - else: - runner = TextTestRunner(verbosity=verbosity) - success = runner.run(test_suite).wasSuccessful() - if not success: - LOGGER.error("Some tests failed. Please see log above.") - return success + base = Path(__file__).parent / "tests" + test_suite_expanded = [str(base / f"{file}.py") for file in test_suite] + + pytest_args = ["--verbose"] if verbose else [] + return 0 == pytest.main([*test_suite_expanded, *pytest_args]) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Run ReadAlongs/Studio test suites.") - parser.add_argument("--quiet", "-q", action="store_true", help="reduce output") - parser.add_argument( - "--describe", action="store_true", help="describe the selected test suite" - ) + parser.add_argument("--verbose", "-v", action="store_true", help="verbose output") parser.add_argument( "suite", nargs="?", @@ -159,6 +73,6 @@ def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool: choices=SUITES, ) args = parser.parse_args() - result = run_tests(args.suite, args.describe, 1 if args.quiet else 3) + result = run_tests(args.suite, args.verbose) if not result: sys.exit(1) From b4b40542dba0a8d6b27cfab00f65906fcc67fe1e Mon Sep 17 00:00:00 2001 From: Eric Joanis Date: Tue, 21 Apr 2026 17:48:16 -0400 Subject: [PATCH 5/5] ci: heroku workflow also needs pytest --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a9c976ed..1a0f31b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -157,8 +157,8 @@ jobs: curl http://127.0.0.1:8000/api/v1/langs | grep Cree kill %1 - - name: Install test dependency - run: pip install httpx + - name: Install minimal test dependencies + run: pip install httpx pytest - name: unit test the web API run: python -m tests.test_web_api