Add mypy type checking to CI + upgrade to Python 3.11 in CI

.semaphore/semaphore.yml

@@ -196,14 +196,14 @@ blocks:
       jobs:
         - name: Build and Tests with 'classic' group protocol
           commands:
-            - sem-version python 3.9
+            - sem-version python 3.11
             # use a virtualenv
             - python3 -m venv _venv && source _venv/bin/activate
             - chmod u+r+x tools/source-package-verification.sh
             - tools/source-package-verification.sh
         - name: Build and Tests with 'consumer' group protocol
           commands:
-            - sem-version python 3.9
+            - sem-version python 3.11
             - sem-version java 17
             # use a virtualenv
             - python3 -m venv _venv && source _venv/bin/activate
@@ -212,7 +212,7 @@ blocks:
             - tools/source-package-verification.sh
         - name: Build, Test, and Report coverage
           commands:
-            - sem-version python 3.9
+            - sem-version python 3.11
             # use a virtualenv
             - python3 -m venv _venv && source _venv/bin/activate
             - chmod u+r+x tools/source-package-verification.sh
@@ -236,7 +236,7 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - sem-version python 3.9
+            - sem-version python 3.11
             # use a virtualenv
             - python3 -m venv _venv && source _venv/bin/activate
             - chmod u+r+x tools/source-package-verification.sh
@@ -255,7 +255,7 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - sem-version python 3.9
+            - sem-version python 3.11
             # use a virtualenv
             - python3 -m venv _venv && source _venv/bin/activate
             - chmod u+r+x tools/source-package-verification.sh
@@ -274,7 +274,7 @@ blocks:
       jobs:
         - name: Build
           commands:
-            - sem-version python 3.9
+            - sem-version python 3.11
             # use a virtualenv
             - python3 -m venv _venv && source _venv/bin/activate
             - chmod u+r+x tools/source-package-verification.sh
@@ -301,7 +301,7 @@ blocks:
         - name: Build and Tests
           commands:
             # Setup Python environment
-            - sem-version python 3.9
+            - sem-version python 3.11
             - python3 -m venv _venv && source _venv/bin/activate
 
             # Install ducktape framework and additional dependencies

MANIFEST.in

@@ -1,5 +1,7 @@
 include README.md
 include LICENSE
 include src/confluent_kafka/src/*.[ch]
+include src/confluent_kafka/py.typed
+include src/confluent_kafka/cimpl.pyi
 prune tests
-prune docs
+prune docs

Makefile

@@ -2,6 +2,9 @@ all:
 	@echo "Targets:"
 	@echo " clean"
 	@echo " docs"
+	@echo " mypy"
+	@echo " style-check"
+	@echo " style-fix"
 
 
 clean:
@@ -14,6 +17,9 @@ clean:
 docs:
 	$(MAKE) -C docs html
 
+mypy:
+	python3 -m mypy src/confluent_kafka
+
 style-check:
 	@(tools/style-format.sh \
 		$$(git ls-tree -r --name-only HEAD | egrep '\.(c|h|py)$$') )

requirements/requirements-avro.txt

@@ -1,4 +1,4 @@
 fastavro < 1.8.0; python_version == "3.7"
 fastavro < 2; python_version > "3.7"
 requests
-avro>=1.11.1,<2
+avro>=1.11.1,<2

requirements/requirements-tests.txt

@@ -3,6 +3,7 @@ urllib3<3
 flake8
 mypy
 types-cachetools
+types-requests
 orjson
 pytest
 pytest-timeout

src/confluent_kafka/deserializing_consumer.py

@@ -106,7 +106,11 @@ def poll(self, timeout: float = -1) -> Optional[Message]:
         if error is not None:
             raise ConsumeError(error, kafka_message=msg)
 
-        ctx = SerializationContext(msg.topic(), MessageField.VALUE, msg.headers())
+        topic = msg.topic()
+        if topic is None:
+            raise TypeError("Message topic is None")
+        ctx = SerializationContext(topic, MessageField.VALUE, msg.headers())
+
         value = msg.value()
         if self._value_deserializer is not None:
             try:

tools/source-package-verification.sh

@@ -5,6 +5,7 @@
 #
 set -e
 
+pip install --upgrade pip
 pip install -r requirements/requirements-tests-install.txt
 pip install -U build
 
@@ -51,6 +52,10 @@ if [[ $OS_NAME == linux && $ARCH == x64 ]]; then
         # Run these actions and tests only in this case
         echo "Building documentation ..."
         flake8 --exclude ./_venv,*_pb2.py,./build
+
+        echo "Running mypy type checking ..."
+        python3.11 -m mypy src/confluent_kafka
+
         pip install -r requirements/requirements-docs.txt
         make docs
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = flake8,py37,py38,py39,py310,py311,py312,py313
+envlist = flake8,mypy,py37,py38,py39,py310,py311,py312,py313
 
 [testenv]
 passenv =
@@ -20,6 +20,13 @@ commands =
 deps = flake8
 commands = flake8
 
+[testenv:mypy]
+deps =
+    mypy
+    types-cachetools
+    types-requests~=2.32.0
+commands = mypy src/confluent_kafka
+
 [pytest]
 python_files = test_*
 testpaths = tests