From 63f54f5857c4600547ffb305a22c2243e379cf9c Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 12:56:48 +0200 Subject: [PATCH 1/6] Load the native library on first use instead of at import Loading the library starts the Go runtime, and a Go runtime doesn't survive fork(). Because cffi.py loaded it at import time, a child forked from a parent that had only imported tls_client hung (Linux) or crashed (macOS) on its first request. cffi.load_library() now loads it once, on the first call, and request, freeMemory and destroySession are thin wrappers around it, so sessions.py keeps its imports. The README explains when to use the spawn or forkserver start method, and how to keep gevent's hub responsive. Fixes #101 Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_multiprocessing.py | 41 +++++++++++++++++++++++ tests/unit/test_cffi.py | 35 +++++++++++++++++++ tls_client/cffi.py | 37 +++++++++++++------- 3 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 tests/integration/test_multiprocessing.py diff --git a/tests/integration/test_multiprocessing.py b/tests/integration/test_multiprocessing.py new file mode 100644 index 0000000..e6e5f17 --- /dev/null +++ b/tests/integration/test_multiprocessing.py @@ -0,0 +1,41 @@ +import multiprocessing +import subprocess +import sys + +import pytest + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif("fork" not in multiprocessing.get_all_start_methods(), reason="needs the fork start method"), +] + +# The parent imports tls_client but makes no request, then a forked child does. The Go runtime doesn't +# survive fork(), so this hung while the library was loaded at import time. +PARENT = """ +import multiprocessing +import sys + +import tls_client + + +def child(url): + with tls_client.Session() as session: + sys.exit(0 if session.get(url, timeout_seconds=10).status_code == 200 else 1) + + +if __name__ == "__main__": + process = multiprocessing.get_context("fork").Process(target=child, args=(sys.argv[1],)) + process.start() + process.join(30) + if process.is_alive(): + process.kill() + sys.exit("the forked child hung") + sys.exit(process.exitcode) +""" + + +def test_forked_child_can_make_requests_after_parent_imported_tls_client(httpbin): + # A fresh interpreter, because this one has loaded the library already. + result = subprocess.run([sys.executable, "-c", PARENT, f"{httpbin.url}/get"], capture_output=True, timeout=60) + + assert result.returncode == 0, result.stderr.decode() diff --git a/tests/unit/test_cffi.py b/tests/unit/test_cffi.py index c37cfce..7e1f796 100644 --- a/tests/unit/test_cffi.py +++ b/tests/unit/test_cffi.py @@ -1,3 +1,6 @@ +import subprocess +import sys + import pytest from tls_client import cffi @@ -86,3 +89,35 @@ def test_missing_library_says_how_to_download_it(monkeypatch, tmp_path): with pytest.raises(FileNotFoundError, match=r"python scripts/fetch_library\.py tls-client-"): cffi._load_library() + + +def test_importing_tls_client_does_not_load_the_library(): + # A fresh interpreter, because this one may have loaded the library already. + code = ( + "import ctypes\n" + "def refuse(path): raise AssertionError(f'loaded {path} too early')\n" + "ctypes.cdll.LoadLibrary = refuse\n" + "import tls_client\n" + "tls_client.Session()\n" + ) + + subprocess.run([sys.executable, "-c", code], check=True, timeout=60) + + +def test_library_is_loaded_once(monkeypatch): + loaded = [] + + def load(): + loaded.append(True) + return "library" + + monkeypatch.setattr(cffi, "_load_library", load) + cffi.load_library.cache_clear() + try: + assert cffi.load_library() == "library" + assert cffi.load_library() == "library" + finally: + # Later tests should get the real library. + cffi.load_library.cache_clear() + + assert loaded == [True] diff --git a/tls_client/cffi.py b/tls_client/cffi.py index 6164fe9..362453b 100644 --- a/tls_client/cffi.py +++ b/tls_client/cffi.py @@ -1,4 +1,5 @@ import ctypes +import functools import os import platform import sys @@ -72,20 +73,32 @@ def _load_library() -> ctypes.CDLL: "Wheels on PyPI include it, so this usually means tls_client runs from a source checkout. " f"Download it with: python scripts/fetch_library.py {filename}" ) - return ctypes.cdll.LoadLibrary(path) + library = ctypes.cdll.LoadLibrary(path) + for function in (library.request, library.freeMemory, library.destroySession): + function.argtypes = [ctypes.c_char_p] + function.restype = ctypes.c_char_p + return library -library = _load_library() +@functools.cache +def load_library() -> ctypes.CDLL: + """The native library, loaded on first use. -# extract the exposed request function from the shared package -request = library.request -request.argtypes = [ctypes.c_char_p] -request.restype = ctypes.c_char_p + Loading it starts the Go runtime, which doesn't survive fork(): a forked child that calls into a runtime + started by its parent deadlocks. Loading lazily keeps `import tls_client` safe before forking. + """ + return _load_library() + + +def request(payload: bytes) -> bytes: + response: bytes = load_library().request(payload) + return response + + +def freeMemory(response_id: bytes) -> None: + load_library().freeMemory(response_id) -freeMemory = library.freeMemory -freeMemory.argtypes = [ctypes.c_char_p] -freeMemory.restype = ctypes.c_char_p -destroySession = library.destroySession -destroySession.argtypes = [ctypes.c_char_p] -destroySession.restype = ctypes.c_char_p +def destroySession(payload: bytes) -> bytes: + response: bytes = load_library().destroySession(payload) + return response From 3c8487483f7f95a83ccd91dc7f113e8d790c14b7 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 12:57:02 +0200 Subject: [PATCH 2/6] Declare freeMemory as returning nothing The Go function returns void, but its restype was c_char_p, so ctypes read whatever was left in the return register as a string pointer and copied the bytes it pointed to. Co-Authored-By: Claude Opus 5.5 --- tls_client/cffi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tls_client/cffi.py b/tls_client/cffi.py index 362453b..b893415 100644 --- a/tls_client/cffi.py +++ b/tls_client/cffi.py @@ -77,6 +77,8 @@ def _load_library() -> ctypes.CDLL: for function in (library.request, library.freeMemory, library.destroySession): function.argtypes = [ctypes.c_char_p] function.restype = ctypes.c_char_p + # freeMemory returns nothing; reading its return register as a string pointer could crash. + library.freeMemory.restype = None return library From 0efbc61858debb8606c9cc82c69900b289abbbae Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 12:58:36 +0200 Subject: [PATCH 3/6] Pin library checksums in SHA256SUMS and document where the libraries come from The pinned hashes move from fetch_library.py into tls_client/dependencies/SHA256SUMS, in sha256sum format, so it ships in every wheel and the sdist and anyone can check an installed library with sha256sum --check. fetch_library.py --update-checksums regenerates it: it downloads every library of the release, checks it against the digest GitHub lists for the asset (upstream publishes no checksum file), and fails on a mismatch. The README gains a section saying the libraries are unmodified assets of the bogdanfinn/tls-client v1.16.0 release, which asset each one is, and how to verify and update them. Refs #34 Co-Authored-By: Claude Opus 5.5 --- README.md | 19 ++++++ scripts/fetch_library.py | 100 +++++++++++++++++++++-------- tls_client/dependencies/SHA256SUMS | 6 ++ 3 files changed, 98 insertions(+), 27 deletions(-) create mode 100644 tls_client/dependencies/SHA256SUMS diff --git a/README.md b/README.md index 0d2fc0e..e7e53a8 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,25 @@ Windows / 32-bit: --add-binary '{path_to_library}/tls_client/dependencies/tls-client-windows-32.dll;tls_client/dependencies' ``` +# Native library +The TLS and HTTP work is done by [bogdanfinn/tls-client](https://github.com/bogdanfinn/tls-client), a Go library that its author builds into shared libraries with [`cffi_dist/build.sh`](https://github.com/bogdanfinn/tls-client/blob/v1.16.0/cffi_dist/build.sh). This project compiles nothing: each wheel contains one unmodified asset from the upstream [v1.16.0 release](https://github.com/bogdanfinn/tls-client/releases/tag/v1.16.0), renamed without the version. + +| Library | Upstream release asset | +| --- | --- | +| `tls-client-darwin-amd64.dylib` | `tls-client-darwin-amd64-1.16.0.dylib` | +| `tls-client-darwin-arm64.dylib` | `tls-client-darwin-arm64-1.16.0.dylib` | +| `tls-client-linux-arm64.so` | `tls-client-linux-arm64-1.16.0.so` | +| `tls-client-linux-ubuntu-amd64.so` | `tls-client-linux-ubuntu-amd64-1.16.0.so` | +| `tls-client-windows-32.dll` | `tls-client-windows-32-1.16.0.dll` | +| `tls-client-windows-64.dll` | `tls-client-windows-64-1.16.0.dll` | + +[`tls_client/dependencies/SHA256SUMS`](tls_client/dependencies/SHA256SUMS) lists the SHA-256 of every library, and each one matches the digest GitHub shows for the release asset. To check an installed library, run this in the installed `tls_client/dependencies` directory (on macOS, `shasum -a 256` instead of `sha256sum`): +``` +sha256sum --check --ignore-missing SHA256SUMS +``` + +A source checkout has no library; download the one for your platform with `python scripts/fetch_library.py `, which checks it against `SHA256SUMS`. To move to a new upstream release, change `VERSION` in `scripts/fetch_library.py` and run `python scripts/fetch_library.py --update-checksums`: it downloads every library, checks it against the digest GitHub lists for the asset, and rewrites `SHA256SUMS`. Upstream doesn't publish a checksum file of its own. + # Acknowledgements Big shout out to [Bogdanfinn](https://github.com/bogdanfinn) for open sourcing his [tls-client](https://github.com/bogdanfinn/tls-client) in Golang. Also I wanted to keep the syntax as similar as possible to [requests](https://github.com/psf/requests), as most people use it and are familiar with it! diff --git a/scripts/fetch_library.py b/scripts/fetch_library.py index 8d7b34f..9df02c6 100644 --- a/scripts/fetch_library.py +++ b/scripts/fetch_library.py @@ -2,63 +2,109 @@ Usage: python scripts/fetch_library.py tls-client-darwin-arm64.dylib [more filenames...] -Without arguments, lists the available filenames. If tls_client fails to import because its library is -missing, the error message names the file this machine needs. - -To move to a new upstream release, update VERSION and every hash in SHA256. GitHub lists the digests on -the release page, or run: gh release view v -R bogdanfinn/tls-client --json assets +Without arguments, lists the available filenames. If tls_client can't load its library because it is +missing, the error message names the file this machine needs. Every download is checked against +tls_client/dependencies/SHA256SUMS. + +To move to a new upstream release, update VERSION (and the version in the README), then run: +python scripts/fetch_library.py --update-checksums +It downloads every library in ASSETS, checks each against the SHA-256 digest GitHub lists for the +release asset, and rewrites SHA256SUMS. Upstream publishes no checksum file of its own. """ import hashlib +import json import sys import urllib.request from pathlib import Path VERSION = "1.16.0" DOWNLOAD_URL = "https://github.com/bogdanfinn/tls-client/releases/download/v{version}/{asset}" - -# Local filename -> SHA-256 of the upstream asset for VERSION. -SHA256 = { - "tls-client-darwin-amd64.dylib": "6463457ea713a96b3b8c94fd9d8746e7bc510cb6784fcf0f4bb64d9c83e3251a", - "tls-client-darwin-arm64.dylib": "99984d013921c753ab29d28720cb099eff6adf63538347e13652cb5cfe5bdc02", - "tls-client-linux-arm64.so": "e398622f99c0ce8fccb50ff6e414f373b5932a0277ece90468a796992f0ae518", - "tls-client-linux-ubuntu-amd64.so": "2ec853496634545e7a7ea028715763948d55bbdd97aca7ecaa9fea8c2ebb08df", - "tls-client-windows-32.dll": "5203a36f80ea3f9cdfa43bf072a775702d1802acb8e4304e78905210f67dd2af", - "tls-client-windows-64.dll": "53dca636b32d965ee6fe4562f39df959b2063febaa3d740efa925d1873cf11d7", +RELEASE_API_URL = "https://api.github.com/repos/bogdanfinn/tls-client/releases/tags/v{version}" + +# Local filename -> upstream release asset. +ASSETS = { + "tls-client-darwin-amd64.dylib": "tls-client-darwin-amd64-{version}.dylib", + "tls-client-darwin-arm64.dylib": "tls-client-darwin-arm64-{version}.dylib", + "tls-client-linux-arm64.so": "tls-client-linux-arm64-{version}.so", + "tls-client-linux-ubuntu-amd64.so": "tls-client-linux-ubuntu-amd64-{version}.so", + "tls-client-windows-32.dll": "tls-client-windows-32-{version}.dll", + "tls-client-windows-64.dll": "tls-client-windows-64-{version}.dll", } DEPENDENCIES_DIR = Path(__file__).resolve().parent.parent / "tls_client" / "dependencies" +# One " " line per library, as written by sha256sum. +CHECKSUMS = DEPENDENCIES_DIR / "SHA256SUMS" def asset_name(filename: str) -> str: - """tls-client-darwin-arm64.dylib -> tls-client-darwin-arm64-1.16.0.dylib""" - stem, ext = filename.rsplit(".", 1) - return f"{stem}-{VERSION}.{ext}" + return ASSETS[filename].format(version=VERSION) -def fetch(filename: str) -> None: +def download(filename: str) -> bytes: url = DOWNLOAD_URL.format(version=VERSION, asset=asset_name(filename)) with urllib.request.urlopen(url) as response: - data = response.read() + return response.read() + + +def read_checksums() -> dict[str, str]: + checksums = {} + for line in CHECKSUMS.read_text().splitlines(): + digest, filename = line.split() + checksums[filename] = digest + return checksums + +def fetch(filename: str, expected: str) -> None: + data = download(filename) digest = hashlib.sha256(data).hexdigest() - if digest != SHA256[filename]: - raise SystemExit(f"Checksum mismatch for {url}: expected {SHA256[filename]}, got {digest}") + if digest != expected: + raise SystemExit(f"Checksum mismatch for {asset_name(filename)}: expected {expected}, got {digest}") (DEPENDENCIES_DIR / filename).write_bytes(data) print(f"Downloaded {filename} ({len(data) / 1e6:.1f} MB) from tls-client v{VERSION}") -def main(filenames: list[str]) -> None: - unknown = [name for name in filenames if name not in SHA256] - if not filenames or unknown: +def update_checksums() -> None: + with urllib.request.urlopen(RELEASE_API_URL.format(version=VERSION)) as response: + release = json.load(response) + # GitHub computes a digest ("sha256:") when an asset is uploaded; assets from before 2025 have none. + published = {asset["name"]: asset.get("digest") for asset in release["assets"]} + + lines = [] + for filename in sorted(ASSETS): + asset = asset_name(filename) + if asset not in published: + raise SystemExit(f"tls-client v{VERSION} has no release asset {asset}") + digest = hashlib.sha256(download(filename)).hexdigest() + if published[asset] is None: + print(f"Warning: GitHub lists no digest for {asset}, pinning the downloaded file as is", file=sys.stderr) + elif published[asset] != f"sha256:{digest}": + raise SystemExit(f"Checksum mismatch for {asset}: GitHub lists {published[asset]}, got sha256:{digest}") + lines.append(f"{digest} {filename}\n") + print(f"Checked {asset}") + + CHECKSUMS.write_text("".join(lines)) + print(f"Wrote {len(lines)} checksums for tls-client v{VERSION} to {CHECKSUMS}") + + +def main(args: list[str]) -> None: + if args == ["--update-checksums"]: + update_checksums() + return + + unknown = [name for name in args if name not in ASSETS] + if not args or unknown: if unknown: print(f"Unknown library: {', '.join(unknown)}", file=sys.stderr) - print("Available libraries:", *SHA256, sep="\n ", file=sys.stderr) + print("Available libraries:", *ASSETS, sep="\n ", file=sys.stderr) raise SystemExit(2) - for filename in filenames: - fetch(filename) + checksums = read_checksums() + for filename in args: + if filename not in checksums: + raise SystemExit(f"{CHECKSUMS} has no checksum for {filename}; run with --update-checksums") + fetch(filename, checksums[filename]) if __name__ == "__main__": diff --git a/tls_client/dependencies/SHA256SUMS b/tls_client/dependencies/SHA256SUMS new file mode 100644 index 0000000..b4c83ba --- /dev/null +++ b/tls_client/dependencies/SHA256SUMS @@ -0,0 +1,6 @@ +6463457ea713a96b3b8c94fd9d8746e7bc510cb6784fcf0f4bb64d9c83e3251a tls-client-darwin-amd64.dylib +99984d013921c753ab29d28720cb099eff6adf63538347e13652cb5cfe5bdc02 tls-client-darwin-arm64.dylib +e398622f99c0ce8fccb50ff6e414f373b5932a0277ece90468a796992f0ae518 tls-client-linux-arm64.so +2ec853496634545e7a7ea028715763948d55bbdd97aca7ecaa9fea8c2ebb08df tls-client-linux-ubuntu-amd64.so +5203a36f80ea3f9cdfa43bf072a775702d1802acb8e4304e78905210f67dd2af tls-client-windows-32.dll +53dca636b32d965ee6fe4562f39df959b2063febaa3d740efa925d1873cf11d7 tls-client-windows-64.dll From aacee558dee679d85a0b21bb039447c529cb46dd Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:00:01 +0200 Subject: [PATCH 4/6] Add the 32-bit x86 and armv7 Linux libraries from tls-client v1.16.0 Two more upstream assets, with their checksums in SHA256SUMS: tls-client-linux-386.so <- tls-client-xgo-1.16.0-linux-386.so tls-client-linux-armv7.so <- tls-client-linux-armv7-1.16.0.so Upstream's own builds have no 32-bit x86 Linux library, so that one is its xgo build. Both are glibc libraries needing at most GLIBC_2.4; the 386 build is GO386=sse2, the armv7 build GOARM=7 with the hard-float ABI. They become manylinux_2_17_i686 and manylinux_2_17_armv7l wheels. GitHub has no 32-bit runners, so the Wheels workflow smoke-tests them in linux/386 and linux/arm/v7 (QEMU) python:3.12-slim containers. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/wheels.yml | 34 +++++++++++++++++++++++++++++- README.md | 4 +++- hatch_build.py | 7 ++++-- scripts/fetch_library.py | 3 +++ tls_client/dependencies/SHA256SUMS | 2 ++ 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 7d6c4be..7777a81 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -47,7 +47,9 @@ jobs: library: - tls-client-darwin-amd64.dylib - tls-client-darwin-arm64.dylib + - tls-client-linux-386.so - tls-client-linux-arm64.so + - tls-client-linux-armv7.so - tls-client-linux-ubuntu-amd64.so - tls-client-windows-32.dll - tls-client-windows-64.dll @@ -107,10 +109,40 @@ jobs: # uv refuses a wheel whose tags don't match the interpreter, so this also checks the tag. - run: uv run --no-project --isolated --python ${{ matrix.python }} --with dist/*.whl python -I scripts/smoke_test.py ${{ matrix.library }} + smoke-test-container: + name: Smoke test (${{ matrix.library }}) + needs: wheel + # GitHub has no 32-bit runners, so these run in a 32-bit container, under QEMU for armv7. + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - library: tls-client-linux-386.so + platform: linux/386 + - library: tls-client-linux-armv7.so + platform: linux/arm/v7 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + if: matrix.platform == 'linux/arm/v7' + with: + platforms: arm + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: wheel-${{ matrix.library }} + path: dist + # pip refuses a wheel whose tags don't match the interpreter, so this also checks the tag. + - run: >- + docker run --rm --platform ${{ matrix.platform }} --volume "$PWD:/src" --workdir /src python:3.12-slim + sh -c "pip install --quiet --root-user-action=ignore dist/*.whl && python -I scripts/smoke_test.py ${{ matrix.library }}" + publish: name: Publish to PyPI if: github.event_name == 'release' - needs: [sdist, smoke-test] + needs: [sdist, smoke-test, smoke-test-container] runs-on: ubuntu-latest environment: name: pypi diff --git a/README.md b/README.md index e7e53a8..4c298a0 100644 --- a/README.md +++ b/README.md @@ -142,13 +142,15 @@ Windows / 32-bit: ``` # Native library -The TLS and HTTP work is done by [bogdanfinn/tls-client](https://github.com/bogdanfinn/tls-client), a Go library that its author builds into shared libraries with [`cffi_dist/build.sh`](https://github.com/bogdanfinn/tls-client/blob/v1.16.0/cffi_dist/build.sh). This project compiles nothing: each wheel contains one unmodified asset from the upstream [v1.16.0 release](https://github.com/bogdanfinn/tls-client/releases/tag/v1.16.0), renamed without the version. +The TLS and HTTP work is done by [bogdanfinn/tls-client](https://github.com/bogdanfinn/tls-client), a Go library that its author builds into shared libraries with [`cffi_dist/build.sh`](https://github.com/bogdanfinn/tls-client/blob/v1.16.0/cffi_dist/build.sh). This project compiles nothing: each wheel contains one unmodified asset from the upstream [v1.16.0 release](https://github.com/bogdanfinn/tls-client/releases/tag/v1.16.0), renamed as listed below. | Library | Upstream release asset | | --- | --- | | `tls-client-darwin-amd64.dylib` | `tls-client-darwin-amd64-1.16.0.dylib` | | `tls-client-darwin-arm64.dylib` | `tls-client-darwin-arm64-1.16.0.dylib` | +| `tls-client-linux-386.so` | `tls-client-xgo-1.16.0-linux-386.so` | | `tls-client-linux-arm64.so` | `tls-client-linux-arm64-1.16.0.so` | +| `tls-client-linux-armv7.so` | `tls-client-linux-armv7-1.16.0.so` | | `tls-client-linux-ubuntu-amd64.so` | `tls-client-linux-ubuntu-amd64-1.16.0.so` | | `tls-client-windows-32.dll` | `tls-client-windows-32-1.16.0.dll` | | `tls-client-windows-64.dll` | `tls-client-windows-64-1.16.0.dll` | diff --git a/hatch_build.py b/hatch_build.py index 0a75849..55d4379 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -10,12 +10,15 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface # Library filename -> wheel platform tag. The floors come from the upstream binaries: the Linux -# builds need glibc 2.4 (amd64) and 2.17 (arm64), and the macOS builds target macOS 13. There is no -# musl wheel: musl can't dlopen the upstream musl build. +# builds need glibc 2.4 (amd64, i686, armv7) and 2.17 (arm64), and the macOS builds target macOS 13. +# The armv7 build uses the hard-float ABI, which is what armv7l wheel tags mean. There is no musl +# wheel: musl can't dlopen the upstream musl build. PLATFORM_TAGS = { "tls-client-darwin-amd64.dylib": "macosx_13_0_x86_64", "tls-client-darwin-arm64.dylib": "macosx_13_0_arm64", + "tls-client-linux-386.so": "manylinux_2_17_i686", "tls-client-linux-arm64.so": "manylinux_2_17_aarch64", + "tls-client-linux-armv7.so": "manylinux_2_17_armv7l", "tls-client-linux-ubuntu-amd64.so": "manylinux_2_17_x86_64", "tls-client-windows-32.dll": "win32", "tls-client-windows-64.dll": "win_amd64", diff --git a/scripts/fetch_library.py b/scripts/fetch_library.py index 9df02c6..6a104ca 100644 --- a/scripts/fetch_library.py +++ b/scripts/fetch_library.py @@ -26,7 +26,10 @@ ASSETS = { "tls-client-darwin-amd64.dylib": "tls-client-darwin-amd64-{version}.dylib", "tls-client-darwin-arm64.dylib": "tls-client-darwin-arm64-{version}.dylib", + # Upstream's own builds have no 32-bit x86 Linux library; this is its xgo build. + "tls-client-linux-386.so": "tls-client-xgo-{version}-linux-386.so", "tls-client-linux-arm64.so": "tls-client-linux-arm64-{version}.so", + "tls-client-linux-armv7.so": "tls-client-linux-armv7-{version}.so", "tls-client-linux-ubuntu-amd64.so": "tls-client-linux-ubuntu-amd64-{version}.so", "tls-client-windows-32.dll": "tls-client-windows-32-{version}.dll", "tls-client-windows-64.dll": "tls-client-windows-64-{version}.dll", diff --git a/tls_client/dependencies/SHA256SUMS b/tls_client/dependencies/SHA256SUMS index b4c83ba..10ab9a1 100644 --- a/tls_client/dependencies/SHA256SUMS +++ b/tls_client/dependencies/SHA256SUMS @@ -1,6 +1,8 @@ 6463457ea713a96b3b8c94fd9d8746e7bc510cb6784fcf0f4bb64d9c83e3251a tls-client-darwin-amd64.dylib 99984d013921c753ab29d28720cb099eff6adf63538347e13652cb5cfe5bdc02 tls-client-darwin-arm64.dylib +5ea81308e53e236dae73b35bd4a09af1201191c32713ea2968b85bbcfe94b616 tls-client-linux-386.so e398622f99c0ce8fccb50ff6e414f373b5932a0277ece90468a796992f0ae518 tls-client-linux-arm64.so +22baa029d4ee8cf327d10cda0e66c29cf256b3eb48e1fe9d6a76954b66f77711 tls-client-linux-armv7.so 2ec853496634545e7a7ea028715763948d55bbdd97aca7ecaa9fea8c2ebb08df tls-client-linux-ubuntu-amd64.so 5203a36f80ea3f9cdfa43bf072a775702d1802acb8e4304e78905210f67dd2af tls-client-windows-32.dll 53dca636b32d965ee6fe4562f39df959b2063febaa3d740efa925d1873cf11d7 tls-client-windows-64.dll From 1e27aca7330201648abaf57b91db8e8ea80ae6a4 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:13:48 +0200 Subject: [PATCH 5/6] Load the 32-bit x86 and armv7 libraries on 32-bit Linux Linux detection only knew 64-bit x86_64 and aarch64, so 32-bit Pythons (i686, armv7l, and 32-bit userlands on 64-bit kernels such as 32-bit Raspberry Pi OS on a Pi 4) got UnsupportedPlatformError. Before that they silently loaded a 64-bit library. LINUX_LIBRARIES now maps each platform.machine() value to its 64-bit and 32-bit library, and the pointer size picks one: x86_64, i686 and i386 to the amd64 or 386 build; aarch64, armv8l and armv7l to the arm64 or armv7 build. ARMv6 stays unsupported with an explanation, because the armv7 build needs ARMv7 and upstream's ARMv6 build is soft-float. Unsupported platforms now also point to the upstream releases. Fixes #145 Fixes #111 Co-Authored-By: Claude Opus 5.5 --- README.md | 10 ++++++++++ tests/unit/test_cffi.py | 28 +++++++++++++++++++++++----- tls_client/cffi.py | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 61 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 4c298a0..090aefd 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,16 @@ Linux / ARM64: --add-binary '{path_to_library}/tls_client/dependencies/tls-client-linux-arm64.so:tls_client/dependencies' ``` +Linux with glibc / 32-bit x86: +``` +--add-binary '{path_to_library}/tls_client/dependencies/tls-client-linux-386.so:tls_client/dependencies' +``` + +Linux / 32-bit ARMv7 and later (e.g. 32-bit Raspberry Pi OS on a Raspberry Pi 2 or newer): +``` +--add-binary '{path_to_library}/tls_client/dependencies/tls-client-linux-armv7.so:tls_client/dependencies' +``` + macOS on Intel x86 based Macs (pre-M1): ``` --add-binary '{path_to_library}/tls_client/dependencies/tls-client-darwin-amd64.dylib:tls_client/dependencies' diff --git a/tests/unit/test_cffi.py b/tests/unit/test_cffi.py index 7e1f796..dae2bd6 100644 --- a/tests/unit/test_cffi.py +++ b/tests/unit/test_cffi.py @@ -1,5 +1,6 @@ import subprocess import sys +from pathlib import Path import pytest @@ -35,6 +36,14 @@ def pretend( ("darwin", "x86_64", {}, "tls-client-darwin-amd64.dylib"), ("linux", "x86_64", {}, "tls-client-linux-ubuntu-amd64.so"), ("linux", "aarch64", {}, "tls-client-linux-arm64.so"), + # These used to fall through to the x86_64 Alpine library. + ("linux", "i686", {"is_64bit": False}, "tls-client-linux-386.so"), + ("linux", "i386", {"is_64bit": False}, "tls-client-linux-386.so"), + ("linux", "armv7l", {"is_64bit": False}, "tls-client-linux-armv7.so"), + ("linux", "armv8l", {"is_64bit": False}, "tls-client-linux-armv7.so"), + # 32-bit userland on a 64-bit kernel, e.g. 32-bit Raspberry Pi OS on a Raspberry Pi 4. + ("linux", "x86_64", {"is_64bit": False}, "tls-client-linux-386.so"), + ("linux", "aarch64", {"is_64bit": False}, "tls-client-linux-armv7.so"), ("win32", "AMD64", {"sysconfig_platform": "win-amd64"}, "tls-client-windows-64.dll"), # 32-bit Python on 64-bit Windows: the interpreter decides, not the OS. ("win32", "AMD64", {"sysconfig_platform": "win32", "is_64bit": False}, "tls-client-windows-32.dll"), @@ -45,20 +54,21 @@ def test_picks_the_library_for_the_interpreter(host, sys_platform, machine, opti host(sys_platform, machine, **options) assert cffi.library_filename() == expected + # fetch_library.py can download it and a wheel is built for it. + assert expected in (Path(cffi.DEPENDENCIES_DIR) / "SHA256SUMS").read_text().split() @pytest.mark.parametrize( ("sys_platform", "machine", "options"), [ - # These used to fall through to the x86_64 Alpine library. - ("linux", "armv7l", {"is_64bit": False}), - ("linux", "i686", {"is_64bit": False}), ("linux", "riscv64", {}), - # 32-bit userland on a 64-bit kernel. - ("linux", "x86_64", {"is_64bit": False}), + # The armv7 library needs ARMv7, and the 386 one SSE2, which i586 CPUs lack. + ("linux", "armv6l", {"is_64bit": False}), + ("linux", "i586", {"is_64bit": False}), # musl can't dlopen the upstream libraries. ("linux", "x86_64", {"musl": True}), ("linux", "aarch64", {"musl": True}), + ("linux", "armv7l", {"is_64bit": False, "musl": True}), # This used to load the x86_64 DLL, which an arm64 interpreter can't. ("win32", "ARM64", {"sysconfig_platform": "win-arm64"}), ("freebsd14", "amd64", {}), @@ -81,9 +91,17 @@ def test_unsupported_platform_error_lists_the_supported_platforms(host): assert "linux (musl) on aarch64 (64-bit Python)" in message assert "musl-based systems such as Alpine can't load the native library" in message assert all(supported in message for supported in cffi.SUPPORTED_PLATFORMS) + assert "https://github.com/bogdanfinn/tls-client/releases" in message assert isinstance(excinfo.value, TLSClientException) +def test_armv6_error_explains_that_the_arm_library_needs_armv7(host): + host("linux", "armv6l", is_64bit=False) + + with pytest.raises(UnsupportedPlatformError, match=r"The 32-bit ARM library needs ARMv7 or later"): + cffi.library_filename() + + def test_missing_library_says_how_to_download_it(monkeypatch, tmp_path): monkeypatch.setattr(cffi, "DEPENDENCIES_DIR", str(tmp_path)) diff --git a/tls_client/cffi.py b/tls_client/cffi.py index b893415..5ba1900 100644 --- a/tls_client/cffi.py +++ b/tls_client/cffi.py @@ -10,10 +10,23 @@ # Platforms with a prebuilt upstream library, as named in the wheels. SUPPORTED_PLATFORMS = ( "macOS on x86_64 or arm64", - "Linux with glibc on x86_64 or aarch64", + "Linux with glibc on x86_64, aarch64, i686 or armv7l (hard-float)", "Windows on x86 or x86_64", ) +# Linux machine() -> (library for 64-bit Python, library for 32-bit Python). machine() reports the kernel's +# architecture, so a 32-bit userland on a 64-bit kernel still says x86_64 or aarch64. +LINUX_LIBRARIES = { + "x86_64": ("linux-ubuntu-amd64.so", "linux-386.so"), + "amd64": ("linux-ubuntu-amd64.so", "linux-386.so"), + "i686": (None, "linux-386.so"), + "i386": (None, "linux-386.so"), + "aarch64": ("linux-arm64.so", "linux-armv7.so"), + "arm64": ("linux-arm64.so", "linux-armv7.so"), + "armv8l": (None, "linux-armv7.so"), + "armv7l": (None, "linux-armv7.so"), +} + DEPENDENCIES_DIR = os.path.join(os.path.abspath(os.path.dirname(__file__)), "dependencies") @@ -45,21 +58,26 @@ def library_filename() -> str: elif sys.platform == "cygwin": if machine in ("x86_64", "i686", "i386"): target = "windows-64.dll" if is_64bit else "windows-32.dll" - elif sys.platform.startswith("linux") and is_64bit and not _is_musl(): - # A 32-bit userland on a 64-bit kernel still reports the kernel's machine, hence the is_64bit check. - if machine in ("x86_64", "amd64"): - target = "linux-ubuntu-amd64.so" - elif machine in ("aarch64", "arm64"): - target = "linux-arm64.so" + elif sys.platform.startswith("linux") and not _is_musl(): + library_64bit, library_32bit = LINUX_LIBRARIES.get(machine, (None, None)) + target = library_64bit if is_64bit else library_32bit if target is None: musl = _is_musl() bits = "64" if is_64bit else "32" - # Upstream does publish a musl build, but musl refuses to dlopen it (Go c-shared uses initial-exec TLS). - reason = " musl-based systems such as Alpine can't load the native library." if musl else "" + reason = "" + if musl: + # Upstream does publish a musl build, but musl refuses to dlopen it (Go c-shared uses initial-exec TLS). + reason = " musl-based systems such as Alpine can't load the native library." + elif machine.startswith(("armv5", "armv6")): + # Upstream's ARMv5 and ARMv6 builds are soft-float, which a hard-float Python such as Raspberry Pi OS's + # can't load, and the armv7 build stops the process on older CPUs. + reason = " The 32-bit ARM library needs ARMv7 or later, so the Raspberry Pi 1 and Zero can't use it." raise UnsupportedPlatformError( f"tls_client has no native library for {sys.platform}{' (musl)' if musl else ''} on {platform.machine()} " - f"({bits}-bit Python).{reason} Supported platforms: {'; '.join(SUPPORTED_PLATFORMS)}." + f"({bits}-bit Python).{reason} Supported platforms: {'; '.join(SUPPORTED_PLATFORMS)}. " + "If https://github.com/bogdanfinn/tls-client/releases has a build for this platform, " + "please open an issue at https://github.com/FlorianREGAZ/Python-Tls-Client/issues." ) return f"tls-client-{target}" From 0ab27515710b6ef49eebe39c01819b99c2d7de88 Mon Sep 17 00:00:00 2001 From: FlorianRegaz Date: Sat, 10 Oct 2026 13:37:39 +0200 Subject: [PATCH 6/6] Drop the README changes Co-Authored-By: Claude Opus 5.5 --- README.md | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/README.md b/README.md index 090aefd..0d2fc0e 100644 --- a/README.md +++ b/README.md @@ -121,16 +121,6 @@ Linux / ARM64: --add-binary '{path_to_library}/tls_client/dependencies/tls-client-linux-arm64.so:tls_client/dependencies' ``` -Linux with glibc / 32-bit x86: -``` ---add-binary '{path_to_library}/tls_client/dependencies/tls-client-linux-386.so:tls_client/dependencies' -``` - -Linux / 32-bit ARMv7 and later (e.g. 32-bit Raspberry Pi OS on a Raspberry Pi 2 or newer): -``` ---add-binary '{path_to_library}/tls_client/dependencies/tls-client-linux-armv7.so:tls_client/dependencies' -``` - macOS on Intel x86 based Macs (pre-M1): ``` --add-binary '{path_to_library}/tls_client/dependencies/tls-client-darwin-amd64.dylib:tls_client/dependencies' @@ -151,27 +141,6 @@ Windows / 32-bit: --add-binary '{path_to_library}/tls_client/dependencies/tls-client-windows-32.dll;tls_client/dependencies' ``` -# Native library -The TLS and HTTP work is done by [bogdanfinn/tls-client](https://github.com/bogdanfinn/tls-client), a Go library that its author builds into shared libraries with [`cffi_dist/build.sh`](https://github.com/bogdanfinn/tls-client/blob/v1.16.0/cffi_dist/build.sh). This project compiles nothing: each wheel contains one unmodified asset from the upstream [v1.16.0 release](https://github.com/bogdanfinn/tls-client/releases/tag/v1.16.0), renamed as listed below. - -| Library | Upstream release asset | -| --- | --- | -| `tls-client-darwin-amd64.dylib` | `tls-client-darwin-amd64-1.16.0.dylib` | -| `tls-client-darwin-arm64.dylib` | `tls-client-darwin-arm64-1.16.0.dylib` | -| `tls-client-linux-386.so` | `tls-client-xgo-1.16.0-linux-386.so` | -| `tls-client-linux-arm64.so` | `tls-client-linux-arm64-1.16.0.so` | -| `tls-client-linux-armv7.so` | `tls-client-linux-armv7-1.16.0.so` | -| `tls-client-linux-ubuntu-amd64.so` | `tls-client-linux-ubuntu-amd64-1.16.0.so` | -| `tls-client-windows-32.dll` | `tls-client-windows-32-1.16.0.dll` | -| `tls-client-windows-64.dll` | `tls-client-windows-64-1.16.0.dll` | - -[`tls_client/dependencies/SHA256SUMS`](tls_client/dependencies/SHA256SUMS) lists the SHA-256 of every library, and each one matches the digest GitHub shows for the release asset. To check an installed library, run this in the installed `tls_client/dependencies` directory (on macOS, `shasum -a 256` instead of `sha256sum`): -``` -sha256sum --check --ignore-missing SHA256SUMS -``` - -A source checkout has no library; download the one for your platform with `python scripts/fetch_library.py `, which checks it against `SHA256SUMS`. To move to a new upstream release, change `VERSION` in `scripts/fetch_library.py` and run `python scripts/fetch_library.py --update-checksums`: it downloads every library, checks it against the digest GitHub lists for the asset, and rewrites `SHA256SUMS`. Upstream doesn't publish a checksum file of its own. - # Acknowledgements Big shout out to [Bogdanfinn](https://github.com/bogdanfinn) for open sourcing his [tls-client](https://github.com/bogdanfinn/tls-client) in Golang. Also I wanted to keep the syntax as similar as possible to [requests](https://github.com/psf/requests), as most people use it and are familiar with it!