use pytest in CI, and exclude tests from coverage again

.github/workflows/tests.yml

@@ -39,8 +39,8 @@ jobs:
 
       - name: Run tests
         run: |
-          coverage run --parallel-mode run_tests.py prod
-          DEVELOPMENT=1 coverage run --parallel-mode -m tests.test_web_api
+          coverage run -m pytest
+          DEVELOPMENT=1 coverage run tests/test_web_api.py
           coverage combine
           coverage xml
 
@@ -157,8 +157,8 @@ jobs:
           curl http://127.0.0.1:8000/api/v1/langs | grep Cree
           kill %1
 
-      - name: Install test dependency
-        run: pip install httpx
+      - name: Install minimal test dependencies
+        run: pip install httpx pytest
 
       - name: unit test the web API
         run: python -m tests.test_web_api

pyproject.toml

@@ -129,6 +129,16 @@ Repository = "https://github.com/ReadAlongs/Studio"
 Issues = "https://github.com/ReadAlongs/Studio/issues"
 Changelog = "https://github.com/ReadAlongs/Studio/releases"
 
+[tool.coverage.run]
+source_pkgs = ["readalongs"]
+branch = true
+parallel = true
+omit = ["readalongs/waveform2svg/*", "readalongs/epub/*"]
+exclude_also = ["if 0:", "if __name__ == .__main__.:"]
+
+[tool.coverage.report]
+precision = 2
+
 [tool.mypy]
 plugins = ["pydantic.mypy"]
 ignore_missing_imports = true
@@ -140,4 +150,5 @@ profile = "black"
 filterwarnings = [
     "ignore:'audioop' is deprecated and slated for removal in Python 3.13:DeprecationWarning",
     "ignore:.*codecs.open.. is deprecated. Use open.. instead.*:DeprecationWarning",
+    "ignore:.*Module already imported so cannot be rewritten; anyio.*",
 ]

run_tests.py

@@ -4,105 +4,37 @@
 Top-level runner for out test suites
 
 Invoke as
-   ./run.py [suite]
+   ./run_tests.py [suite]
 where [suite] can be one of:
    all: run everything, by searching the directory for all test suite files
    prod: synonym for all
-   dev: run the standard development test suite - this is what we do in CI
+   dev: now also a synonym for all
    e2e: run the end-to-end tests
-   other: run the other tests
+   api: run the API-related tests
 """
 
 import argparse
-import os
-import re
 import sys
-from unittest import TestLoader, TestSuite, TextTestRunner
+from pathlib import Path
+
+import pytest
 
 from readalongs.log import LOGGER
-from tests.test_align_cli import TestAlignCli
-from tests.test_anchors import TestAnchors
-from tests.test_api import TestAlignApi
-from tests.test_audio import TestAudio
-from tests.test_config import TestConfig
-from tests.test_dna_text import TestDNAText
-from tests.test_dna_utils import TestDNAUtils
-from tests.test_dtd import TestDTD
-from tests.test_force_align import TestForceAlignment, TestXHTML
-from tests.test_g2p_cli import TestG2pCli
-from tests.test_make_xml_cli import TestMakeXMLCli
-from tests.test_misc import TestMisc
-from tests.test_package_urls import TestPackageURLs
-from tests.test_silence import TestSilence
-from tests.test_smil import TestSmilUtilities
-from tests.test_temp_file import TestTempFile
-from tests.test_tokenize_cli import TestTokenizeCli
-from tests.test_tokenize_xml import TestTokenizer
-from tests.test_web_api import TestWebApi
-
-LOADER = TestLoader()
-
-e2e_tests = [
-    LOADER.loadTestsFromTestCase(test) for test in (TestForceAlignment, TestXHTML)
-]
-
-api_tests = [
-    LOADER.loadTestsFromTestCase(test) for test in [TestWebApi]
-]  # TODO: add some load testing with https://locust.io/
-
-other_tests = [
-    LOADER.loadTestsFromTestCase(test)
-    for test in [
-        TestAnchors,
-        TestConfig,
-        TestDNAText,
-        TestDNAUtils,
-        TestTokenizer,
-        TestTokenizeCli,
-        TestTempFile,
-        TestMakeXMLCli,
-        TestAudio,
-        TestAlignCli,
-        TestAlignApi,
-        TestG2pCli,
-        TestMisc,
-        TestSilence,
-        TestSmilUtilities,
-        TestPackageURLs,
-        TestWebApi,
-        TestDTD,
-    ]
-]
-
-
-def list_tests(suite: TestSuite):
-    for subsuite in suite:
-        for match in re.finditer(r"tests=\[([^][]+)\]>", str(subsuite)):
-            yield from match[1].split(", ")
-
-
-def describe_suite(suite: TestSuite):
-    full_suite = LOADER.discover(os.path.dirname(__file__))
-    full_list = list(list_tests(full_suite))
-    requested_list = list(list_tests(suite))
-    requested_set = set(requested_list)
-    print("Test suite includes:", *sorted(requested_list), sep="\n")
-    print(
-        "\nTest suite excludes:",
-        *sorted(test for test in full_list if test not in requested_set),
-        sep="\n",
-    )
+
+e2e_tests = ["test_force_align", "test_align_cli"]
+
+# TODO: add some load testing with https://locust.io/
+api_tests = ["test_web_api", "test_api"]
 
 
-SUITES = ["all", "dev", "e2e", "prod", "api", "other"]
+SUITES = ["all", "dev", "e2e", "prod", "api"]
 
 
-def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool:
+def run_tests(suite: str, verbose: bool = True) -> bool:
     """Run the specified test suite.
 
     Args:
         suite: one of SUITES, "dev" if the empty string
-        describe: if True, list all the test cases instead of running them.
 
     Returns: True iff success
     """
@@ -112,38 +44,27 @@ def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool:
         suite = "dev"
 
     if suite == "e2e":
-        test_suite = TestSuite(e2e_tests)
+        test_suite = e2e_tests
     elif suite == "api":
-        test_suite = TestSuite(api_tests)
-    elif suite == "dev":
-        test_suite = TestSuite(other_tests + e2e_tests)
-    elif suite in ("prod", "all"):
-        test_suite = LOADER.discover(os.path.dirname(__file__))
-    elif suite == "other":
-        test_suite = TestSuite(other_tests)
+        test_suite = api_tests
+    elif suite in ("prod", "all", "dev"):
+        test_suite = []
     else:
         LOGGER.error(
             "Sorry, you need to select a Test Suite to run, one of: " + " ".join(SUITES)
         )
         return False
 
-    if describe:
-        describe_suite(test_suite)
-        return True
-    else:
-        runner = TextTestRunner(verbosity=verbosity)
-        success = runner.run(test_suite).wasSuccessful()
-        if not success:
-            LOGGER.error("Some tests failed. Please see log above.")
-        return success
+    base = Path(__file__).parent / "tests"
+    test_suite_expanded = [str(base / f"{file}.py") for file in test_suite]
+
+    pytest_args = ["--verbose"] if verbose else []
+    return 0 == pytest.main([*test_suite_expanded, *pytest_args])
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Run ReadAlongs/Studio test suites.")
-    parser.add_argument("--quiet", "-q", action="store_true", help="reduce output")
-    parser.add_argument(
-        "--describe", action="store_true", help="describe the selected test suite"
-    )
+    parser.add_argument("--verbose", "-v", action="store_true", help="verbose output")
     parser.add_argument(
         "suite",
         nargs="?",
@@ -152,6 +73,6 @@ def run_tests(suite: str, describe: bool = False, verbosity=3) -> bool:
         choices=SUITES,
     )
     args = parser.parse_args()
-    result = run_tests(args.suite, args.describe, 1 if args.quiet else 3)
+    result = run_tests(args.suite, args.verbose)
     if not result:
         sys.exit(1)

tests/test_align_cli.py

@@ -6,13 +6,14 @@
 
 import os
 import subprocess
+import sys
 import tempfile
 from os.path import exists, join
 from pathlib import Path
 from typing import Union
-from unittest import main
 
 from lxml.html import fromstring
+from pytest import main
 
 from readalongs._version import READALONG_FILE_FORMAT_VERSION, VERSION
 from readalongs.cli import align, langs
@@ -152,7 +153,6 @@ def test_invoke_align(self) -> None:
                 str(output),
             ],
         )
-        print("dir(result)", dir(results_output_exists))
         self.assertNotEqual(results_output_exists.exit_code, 0)
         self.assertIn(
             "already exists, use -f to overwrite", results_output_exists.output
@@ -651,4 +651,4 @@ def test_ffmpeg_is_present(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_anchors.py

@@ -3,9 +3,11 @@
 """Unit testing for the anchors functionality in readalongs align"""
 
 import os
+import sys
 from contextlib import redirect_stderr
 from io import StringIO
-from unittest import main
+
+from pytest import main
 
 from readalongs.align import align_audio
 from readalongs.log import LOGGER
@@ -101,4 +103,4 @@ def test_anchors_align_modes(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_api.py

@@ -5,11 +5,12 @@
 """
 
 import re
+import sys
 from contextlib import redirect_stderr
 from io import StringIO
-from unittest import main
 
 import click
+from pytest import main
 
 from readalongs import api
 from readalongs.log import LOGGER
@@ -219,4 +220,4 @@ def test_extract_version_from_url(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_audio.py

@@ -3,9 +3,11 @@
 """Test suite for various audio contents handling methods"""
 
 import os
+import sys
 from pathlib import Path
 from subprocess import run
-from unittest import main
+
+from pytest import main
 
 from readalongs.audio_utils import (
     extract_section,
@@ -165,4 +167,4 @@ def test_write_audio_to_file(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_config.py

@@ -4,10 +4,12 @@
 
 import io
 import os
+import sys
 from contextlib import redirect_stderr
-from unittest import TestCase, main
+from unittest import TestCase
 
 from lxml import etree
+from pytest import main
 
 from readalongs.text.add_elements_to_xml import add_images, add_supplementary_xml
 from readalongs.text.util import load_xml
@@ -79,4 +81,4 @@ def test_arbitrary_xml(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_dna_text.py

@@ -2,11 +2,12 @@
 
 """Test handling of DNA text in tokenization"""
 
+import sys
 from contextlib import redirect_stderr
 from io import StringIO
-from unittest import main
 
 from lxml import etree
+from pytest import main
 
 from readalongs.text import tokenize_xml
 from readalongs.text.add_ids_to_xml import add_ids
@@ -163,4 +164,4 @@ def test_dna_word_nested(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])

tests/test_dna_utils.py

@@ -2,7 +2,10 @@
 
 """Test suite for DNA segment manupulation methods"""
 
-from unittest import TestCase, main
+import sys
+from unittest import TestCase
+
+from pytest import main
 
 from readalongs.dna_utils import (
     calculate_adjustment,
@@ -175,4 +178,4 @@ def test_dna_union(self):
 
 
 if __name__ == "__main__":
-    main()
+    main([__file__, *sys.argv])