Fix 32-bit Linux and Raspberry Pi loading, load the library lazily, document binary provenance

.github/workflows/wheels.yml

@@ -47,7 +47,9 @@ jobs:
         library:
           - tls-client-darwin-amd64.dylib
           - tls-client-darwin-arm64.dylib
+          - tls-client-linux-386.so
           - tls-client-linux-arm64.so
+          - tls-client-linux-armv7.so
           - tls-client-linux-ubuntu-amd64.so
           - tls-client-windows-32.dll
           - tls-client-windows-64.dll
@@ -107,10 +109,40 @@ jobs:
       # uv refuses a wheel whose tags don't match the interpreter, so this also checks the tag.
       - run: uv run --no-project --isolated --python ${{ matrix.python }} --with dist/*.whl python -I scripts/smoke_test.py ${{ matrix.library }}
 
+  smoke-test-container:
+    name: Smoke test (${{ matrix.library }})
+    needs: wheel
+    # GitHub has no 32-bit runners, so these run in a 32-bit container, under QEMU for armv7.
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - library: tls-client-linux-386.so
+            platform: linux/386
+          - library: tls-client-linux-armv7.so
+            platform: linux/arm/v7
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
+        if: matrix.platform == 'linux/arm/v7'
+        with:
+          platforms: arm
+      - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+        with:
+          name: wheel-${{ matrix.library }}
+          path: dist
+      # pip refuses a wheel whose tags don't match the interpreter, so this also checks the tag.
+      - run: >-
+          docker run --rm --platform ${{ matrix.platform }} --volume "$PWD:/src" --workdir /src python:3.12-slim
+          sh -c "pip install --quiet --root-user-action=ignore dist/*.whl && python -I scripts/smoke_test.py ${{ matrix.library }}"
+
   publish:
     name: Publish to PyPI
     if: github.event_name == 'release'
-    needs: [sdist, smoke-test]
+    needs: [sdist, smoke-test, smoke-test-container]
     runs-on: ubuntu-latest
     environment:
       name: pypi

hatch_build.py

@@ -10,12 +10,15 @@
 from hatchling.builders.hooks.plugin.interface import BuildHookInterface
 
 # Library filename -> wheel platform tag. The floors come from the upstream binaries: the Linux
-# builds need glibc 2.4 (amd64) and 2.17 (arm64), and the macOS builds target macOS 13. There is no
-# musl wheel: musl can't dlopen the upstream musl build.
+# builds need glibc 2.4 (amd64, i686, armv7) and 2.17 (arm64), and the macOS builds target macOS 13.
+# The armv7 build uses the hard-float ABI, which is what armv7l wheel tags mean. There is no musl
+# wheel: musl can't dlopen the upstream musl build.
 PLATFORM_TAGS = {
     "tls-client-darwin-amd64.dylib": "macosx_13_0_x86_64",
     "tls-client-darwin-arm64.dylib": "macosx_13_0_arm64",
+    "tls-client-linux-386.so": "manylinux_2_17_i686",
     "tls-client-linux-arm64.so": "manylinux_2_17_aarch64",
+    "tls-client-linux-armv7.so": "manylinux_2_17_armv7l",
     "tls-client-linux-ubuntu-amd64.so": "manylinux_2_17_x86_64",
     "tls-client-windows-32.dll": "win32",
     "tls-client-windows-64.dll": "win_amd64",

scripts/fetch_library.py

@@ -2,63 +2,112 @@
 
 Usage: python scripts/fetch_library.py tls-client-darwin-arm64.dylib [more filenames...]
 
-Without arguments, lists the available filenames. If tls_client fails to import because its library is
-missing, the error message names the file this machine needs.
-
-To move to a new upstream release, update VERSION and every hash in SHA256. GitHub lists the digests on
-the release page, or run: gh release view v<VERSION> -R bogdanfinn/tls-client --json assets
+Without arguments, lists the available filenames. If tls_client can't load its library because it is
+missing, the error message names the file this machine needs. Every download is checked against
+tls_client/dependencies/SHA256SUMS.
+
+To move to a new upstream release, update VERSION (and the version in the README), then run:
+python scripts/fetch_library.py --update-checksums
+It downloads every library in ASSETS, checks each against the SHA-256 digest GitHub lists for the
+release asset, and rewrites SHA256SUMS. Upstream publishes no checksum file of its own.
 """
 
 import hashlib
+import json
 import sys
 import urllib.request
 from pathlib import Path
 
 VERSION = "1.16.0"
 DOWNLOAD_URL = "https://github.com/bogdanfinn/tls-client/releases/download/v{version}/{asset}"
-
-# Local filename -> SHA-256 of the upstream asset for VERSION.
-SHA256 = {
-    "tls-client-darwin-amd64.dylib": "6463457ea713a96b3b8c94fd9d8746e7bc510cb6784fcf0f4bb64d9c83e3251a",
-    "tls-client-darwin-arm64.dylib": "99984d013921c753ab29d28720cb099eff6adf63538347e13652cb5cfe5bdc02",
-    "tls-client-linux-arm64.so": "e398622f99c0ce8fccb50ff6e414f373b5932a0277ece90468a796992f0ae518",
-    "tls-client-linux-ubuntu-amd64.so": "2ec853496634545e7a7ea028715763948d55bbdd97aca7ecaa9fea8c2ebb08df",
-    "tls-client-windows-32.dll": "5203a36f80ea3f9cdfa43bf072a775702d1802acb8e4304e78905210f67dd2af",
-    "tls-client-windows-64.dll": "53dca636b32d965ee6fe4562f39df959b2063febaa3d740efa925d1873cf11d7",
+RELEASE_API_URL = "https://api.github.com/repos/bogdanfinn/tls-client/releases/tags/v{version}"
+
+# Local filename -> upstream release asset.
+ASSETS = {
+    "tls-client-darwin-amd64.dylib": "tls-client-darwin-amd64-{version}.dylib",
+    "tls-client-darwin-arm64.dylib": "tls-client-darwin-arm64-{version}.dylib",
+    # Upstream's own builds have no 32-bit x86 Linux library; this is its xgo build.
+    "tls-client-linux-386.so": "tls-client-xgo-{version}-linux-386.so",
+    "tls-client-linux-arm64.so": "tls-client-linux-arm64-{version}.so",
+    "tls-client-linux-armv7.so": "tls-client-linux-armv7-{version}.so",
+    "tls-client-linux-ubuntu-amd64.so": "tls-client-linux-ubuntu-amd64-{version}.so",
+    "tls-client-windows-32.dll": "tls-client-windows-32-{version}.dll",
+    "tls-client-windows-64.dll": "tls-client-windows-64-{version}.dll",
 }
 
 DEPENDENCIES_DIR = Path(__file__).resolve().parent.parent / "tls_client" / "dependencies"
+# One "<sha256>  <filename>" line per library, as written by sha256sum.
+CHECKSUMS = DEPENDENCIES_DIR / "SHA256SUMS"
 
 
 def asset_name(filename: str) -> str:
-    """tls-client-darwin-arm64.dylib -> tls-client-darwin-arm64-1.16.0.dylib"""
-    stem, ext = filename.rsplit(".", 1)
-    return f"{stem}-{VERSION}.{ext}"
+    return ASSETS[filename].format(version=VERSION)
 
 
-def fetch(filename: str) -> None:
+def download(filename: str) -> bytes:
     url = DOWNLOAD_URL.format(version=VERSION, asset=asset_name(filename))
     with urllib.request.urlopen(url) as response:
-        data = response.read()
+        return response.read()
+
+
+def read_checksums() -> dict[str, str]:
+    checksums = {}
+    for line in CHECKSUMS.read_text().splitlines():
+        digest, filename = line.split()
+        checksums[filename] = digest
+    return checksums
+
 
+def fetch(filename: str, expected: str) -> None:
+    data = download(filename)
     digest = hashlib.sha256(data).hexdigest()
-    if digest != SHA256[filename]:
-        raise SystemExit(f"Checksum mismatch for {url}: expected {SHA256[filename]}, got {digest}")
+    if digest != expected:
+        raise SystemExit(f"Checksum mismatch for {asset_name(filename)}: expected {expected}, got {digest}")
 
     (DEPENDENCIES_DIR / filename).write_bytes(data)
     print(f"Downloaded {filename} ({len(data) / 1e6:.1f} MB) from tls-client v{VERSION}")
 
 
-def main(filenames: list[str]) -> None:
-    unknown = [name for name in filenames if name not in SHA256]
-    if not filenames or unknown:
+def update_checksums() -> None:
+    with urllib.request.urlopen(RELEASE_API_URL.format(version=VERSION)) as response:
+        release = json.load(response)
+    # GitHub computes a digest ("sha256:<hex>") when an asset is uploaded; assets from before 2025 have none.
+    published = {asset["name"]: asset.get("digest") for asset in release["assets"]}
+
+    lines = []
+    for filename in sorted(ASSETS):
+        asset = asset_name(filename)
+        if asset not in published:
+            raise SystemExit(f"tls-client v{VERSION} has no release asset {asset}")
+        digest = hashlib.sha256(download(filename)).hexdigest()
+        if published[asset] is None:
+            print(f"Warning: GitHub lists no digest for {asset}, pinning the downloaded file as is", file=sys.stderr)
+        elif published[asset] != f"sha256:{digest}":
+            raise SystemExit(f"Checksum mismatch for {asset}: GitHub lists {published[asset]}, got sha256:{digest}")
+        lines.append(f"{digest}  {filename}\n")
+        print(f"Checked {asset}")
+
+    CHECKSUMS.write_text("".join(lines))
+    print(f"Wrote {len(lines)} checksums for tls-client v{VERSION} to {CHECKSUMS}")
+
+
+def main(args: list[str]) -> None:
+    if args == ["--update-checksums"]:
+        update_checksums()
+        return
+
+    unknown = [name for name in args if name not in ASSETS]
+    if not args or unknown:
         if unknown:
             print(f"Unknown library: {', '.join(unknown)}", file=sys.stderr)
-        print("Available libraries:", *SHA256, sep="\n  ", file=sys.stderr)
+        print("Available libraries:", *ASSETS, sep="\n  ", file=sys.stderr)
         raise SystemExit(2)
 
-    for filename in filenames:
-        fetch(filename)
+    checksums = read_checksums()
+    for filename in args:
+        if filename not in checksums:
+            raise SystemExit(f"{CHECKSUMS} has no checksum for {filename}; run with --update-checksums")
+        fetch(filename, checksums[filename])
 
 
 if __name__ == "__main__":

tests/integration/test_multiprocessing.py

@@ -0,0 +1,41 @@
+import multiprocessing
+import subprocess
+import sys
+
+import pytest
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif("fork" not in multiprocessing.get_all_start_methods(), reason="needs the fork start method"),
+]
+
+# The parent imports tls_client but makes no request, then a forked child does. The Go runtime doesn't
+# survive fork(), so this hung while the library was loaded at import time.
+PARENT = """
+import multiprocessing
+import sys
+
+import tls_client
+
+
+def child(url):
+    with tls_client.Session() as session:
+        sys.exit(0 if session.get(url, timeout_seconds=10).status_code == 200 else 1)
+
+
+if __name__ == "__main__":
+    process = multiprocessing.get_context("fork").Process(target=child, args=(sys.argv[1],))
+    process.start()
+    process.join(30)
+    if process.is_alive():
+        process.kill()
+        sys.exit("the forked child hung")
+    sys.exit(process.exitcode)
+"""
+
+
+def test_forked_child_can_make_requests_after_parent_imported_tls_client(httpbin):
+    # A fresh interpreter, because this one has loaded the library already.
+    result = subprocess.run([sys.executable, "-c", PARENT, f"{httpbin.url}/get"], capture_output=True, timeout=60)
+
+    assert result.returncode == 0, result.stderr.decode()

tests/unit/test_cffi.py

@@ -1,3 +1,7 @@
+import subprocess
+import sys
+from pathlib import Path
+
 import pytest
 
 from tls_client import cffi
@@ -32,6 +36,14 @@ def pretend(
         ("darwin", "x86_64", {}, "tls-client-darwin-amd64.dylib"),
         ("linux", "x86_64", {}, "tls-client-linux-ubuntu-amd64.so"),
         ("linux", "aarch64", {}, "tls-client-linux-arm64.so"),
+        # These used to fall through to the x86_64 Alpine library.
+        ("linux", "i686", {"is_64bit": False}, "tls-client-linux-386.so"),
+        ("linux", "i386", {"is_64bit": False}, "tls-client-linux-386.so"),
+        ("linux", "armv7l", {"is_64bit": False}, "tls-client-linux-armv7.so"),
+        ("linux", "armv8l", {"is_64bit": False}, "tls-client-linux-armv7.so"),
+        # 32-bit userland on a 64-bit kernel, e.g. 32-bit Raspberry Pi OS on a Raspberry Pi 4.
+        ("linux", "x86_64", {"is_64bit": False}, "tls-client-linux-386.so"),
+        ("linux", "aarch64", {"is_64bit": False}, "tls-client-linux-armv7.so"),
         ("win32", "AMD64", {"sysconfig_platform": "win-amd64"}, "tls-client-windows-64.dll"),
         # 32-bit Python on 64-bit Windows: the interpreter decides, not the OS.
         ("win32", "AMD64", {"sysconfig_platform": "win32", "is_64bit": False}, "tls-client-windows-32.dll"),
@@ -42,20 +54,21 @@ def test_picks_the_library_for_the_interpreter(host, sys_platform, machine, opti
     host(sys_platform, machine, **options)
 
     assert cffi.library_filename() == expected
+    # fetch_library.py can download it and a wheel is built for it.
+    assert expected in (Path(cffi.DEPENDENCIES_DIR) / "SHA256SUMS").read_text().split()
 
 
 @pytest.mark.parametrize(
     ("sys_platform", "machine", "options"),
     [
-        # These used to fall through to the x86_64 Alpine library.
-        ("linux", "armv7l", {"is_64bit": False}),
-        ("linux", "i686", {"is_64bit": False}),
         ("linux", "riscv64", {}),
-        # 32-bit userland on a 64-bit kernel.
-        ("linux", "x86_64", {"is_64bit": False}),
+        # The armv7 library needs ARMv7, and the 386 one SSE2, which i586 CPUs lack.
+        ("linux", "armv6l", {"is_64bit": False}),
+        ("linux", "i586", {"is_64bit": False}),
         # musl can't dlopen the upstream libraries.
         ("linux", "x86_64", {"musl": True}),
         ("linux", "aarch64", {"musl": True}),
+        ("linux", "armv7l", {"is_64bit": False, "musl": True}),
         # This used to load the x86_64 DLL, which an arm64 interpreter can't.
         ("win32", "ARM64", {"sysconfig_platform": "win-arm64"}),
         ("freebsd14", "amd64", {}),
@@ -78,11 +91,51 @@ def test_unsupported_platform_error_lists_the_supported_platforms(host):
     assert "linux (musl) on aarch64 (64-bit Python)" in message
     assert "musl-based systems such as Alpine can't load the native library" in message
     assert all(supported in message for supported in cffi.SUPPORTED_PLATFORMS)
+    assert "https://github.com/bogdanfinn/tls-client/releases" in message
     assert isinstance(excinfo.value, TLSClientException)
 
 
+def test_armv6_error_explains_that_the_arm_library_needs_armv7(host):
+    host("linux", "armv6l", is_64bit=False)
+
+    with pytest.raises(UnsupportedPlatformError, match=r"The 32-bit ARM library needs ARMv7 or later"):
+        cffi.library_filename()
+
+
 def test_missing_library_says_how_to_download_it(monkeypatch, tmp_path):
     monkeypatch.setattr(cffi, "DEPENDENCIES_DIR", str(tmp_path))
 
     with pytest.raises(FileNotFoundError, match=r"python scripts/fetch_library\.py tls-client-"):
         cffi._load_library()
+
+
+def test_importing_tls_client_does_not_load_the_library():
+    # A fresh interpreter, because this one may have loaded the library already.
+    code = (
+        "import ctypes\n"
+        "def refuse(path): raise AssertionError(f'loaded {path} too early')\n"
+        "ctypes.cdll.LoadLibrary = refuse\n"
+        "import tls_client\n"
+        "tls_client.Session()\n"
+    )
+
+    subprocess.run([sys.executable, "-c", code], check=True, timeout=60)
+
+
+def test_library_is_loaded_once(monkeypatch):
+    loaded = []
+
+    def load():
+        loaded.append(True)
+        return "library"
+
+    monkeypatch.setattr(cffi, "_load_library", load)
+    cffi.load_library.cache_clear()
+    try:
+        assert cffi.load_library() == "library"
+        assert cffi.load_library() == "library"
+    finally:
+        # Later tests should get the real library.
+        cffi.load_library.cache_clear()
+
+    assert loaded == [True]